From 54b9c5feb052ae8a1ca798bb9b2f98cf3ab30612 Mon Sep 17 00:00:00 2001 From: diml Date: Wed, 19 Dec 2007 21:59:41 +0000 Subject: [PATCH] try update Zend directory to 1.9 equivalent --- search/Zend/Search/Exception.php | 6 +- search/Zend/Search/Lucene.php | 728 ++++++++++++++---- .../Zend/Search/Lucene/Analysis/Analyzer.php | 95 ++- .../Lucene/Analysis/Analyzer/Common.php | 13 +- .../Lucene/Analysis/Analyzer/Common/Text.php | 85 +- .../Analyzer/Common/Text/CaseInsensitive.php | 8 +- .../Analysis/Analyzer/Common/TextNum.php | 92 +++ .../Common/TextNum/CaseInsensitive.php | 46 ++ .../Lucene/Analysis/Analyzer/Common/Utf8.php | 169 ++++ .../Analysis/Analyzer/Common/Utf8Num.php | 169 ++++ search/Zend/Search/Lucene/Analysis/Token.php | 24 +- .../Search/Lucene/Analysis/TokenFilter.php | 6 +- .../Lucene/Analysis/TokenFilter/LowerCase.php | 12 +- .../Analysis/TokenFilter/ShortWords.php | 68 ++ .../Lucene/Analysis/TokenFilter/StopWords.php | 101 +++ search/Zend/Search/Lucene/Document.php | 24 +- search/Zend/Search/Lucene/Document/Html.php | 310 ++++++++ search/Zend/Search/Lucene/Exception.php | 6 +- search/Zend/Search/Lucene/FSM.php | 433 +++++++++++ search/Zend/Search/Lucene/FSMAction.php | 65 ++ search/Zend/Search/Lucene/Field.php | 102 ++- .../Search/Lucene/Index/DictionaryLoader.php | 254 ++++++ search/Zend/Search/Lucene/Index/FieldInfo.php | 4 +- .../Zend/Search/Lucene/Index/SegmentInfo.php | 641 +++++++++++++-- .../Lucene/Index/SegmentInfoPriorityQueue.php | 53 ++ .../Search/Lucene/Index/SegmentMerger.php | 273 +++++++ .../Search/Lucene/Index/SegmentWriter.php | 589 +++++++------- .../Index/SegmentWriter/DocumentWriter.php | 213 +++++ .../Index/SegmentWriter/StreamWriter.php | 94 +++ search/Zend/Search/Lucene/Index/Term.php | 50 +- search/Zend/Search/Lucene/Index/TermInfo.php | 4 +- search/Zend/Search/Lucene/Index/Writer.php | 435 +++++++---- search/Zend/Search/Lucene/Interface.php | 330 ++++++++ search/Zend/Search/Lucene/PriorityQueue.php | 170 ++++ search/Zend/Search/Lucene/Proxy.php | 468 +++++++++++ .../Search/BooleanExpressionRecognizer.php | 280 +++++++ search/Zend/Search/Lucene/Search/Query.php | 149 +++- .../Search/Lucene/Search/Query/Boolean.php | 715 +++++++++++++++++ .../Zend/Search/Lucene/Search/Query/Empty.php | 139 ++++ .../Search/Lucene/Search/Query/MultiTerm.php | 478 ++++++++---- .../Search/Lucene/Search/Query/Phrase.php | 260 +++++-- .../Zend/Search/Lucene/Search/Query/Term.php | 180 ++++- .../Zend/Search/Lucene/Search/QueryEntry.php | 87 +++ .../Lucene/Search/QueryEntry/Phrase.php | 147 ++++ .../Lucene/Search/QueryEntry/Subquery.php | 86 +++ .../Search/Lucene/Search/QueryEntry/Term.php | 154 ++++ search/Zend/Search/Lucene/Search/QueryHit.php | 16 +- .../Zend/Search/Lucene/Search/QueryLexer.php | 508 ++++++++++++ .../Zend/Search/Lucene/Search/QueryParser.php | 533 +++++++++++-- .../Lucene/Search/QueryParserContext.php | 416 ++++++++++ .../Lucene/Search/QueryParserException.php | 40 + .../Zend/Search/Lucene/Search/QueryToken.php | 196 ++++- .../Search/Lucene/Search/QueryTokenizer.php | 4 +- .../Zend/Search/Lucene/Search/Similarity.php | 22 +- .../Lucene/Search/Similarity/Default.php | 8 +- search/Zend/Search/Lucene/Search/Weight.php | 29 +- .../Search/Lucene/Search/Weight/Boolean.php | 136 ++++ .../Search/Lucene/Search/Weight/Empty.php | 56 ++ .../Search/Lucene/Search/Weight/MultiTerm.php | 26 +- .../Search/Lucene/Search/Weight/Phrase.php | 48 +- .../Zend/Search/Lucene/Search/Weight/Term.php | 42 +- .../Zend/Search/Lucene/Storage/Directory.php | 12 +- .../Lucene/Storage/Directory/Filesystem.php | 65 +- search/Zend/Search/Lucene/Storage/File.php | 31 +- .../Search/Lucene/Storage/File/Filesystem.php | 57 +- .../Search/Lucene/Storage/File/Memory.php | 555 +++++++++++++ search/Zend/Search/TODO.txt | 9 +- 67 files changed, 10344 insertions(+), 1280 deletions(-) create mode 100644 search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php create mode 100644 search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php create mode 100644 search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php create mode 100644 search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php create mode 100644 search/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php create mode 100644 search/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php create mode 100644 search/Zend/Search/Lucene/Document/Html.php create mode 100644 search/Zend/Search/Lucene/FSM.php create mode 100644 search/Zend/Search/Lucene/FSMAction.php create mode 100644 search/Zend/Search/Lucene/Index/DictionaryLoader.php create mode 100644 search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php create mode 100644 search/Zend/Search/Lucene/Index/SegmentMerger.php create mode 100644 search/Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php create mode 100644 search/Zend/Search/Lucene/Index/SegmentWriter/StreamWriter.php create mode 100644 search/Zend/Search/Lucene/Interface.php create mode 100644 search/Zend/Search/Lucene/PriorityQueue.php create mode 100644 search/Zend/Search/Lucene/Proxy.php create mode 100644 search/Zend/Search/Lucene/Search/BooleanExpressionRecognizer.php create mode 100644 search/Zend/Search/Lucene/Search/Query/Boolean.php create mode 100644 search/Zend/Search/Lucene/Search/Query/Empty.php create mode 100644 search/Zend/Search/Lucene/Search/QueryEntry.php create mode 100644 search/Zend/Search/Lucene/Search/QueryEntry/Phrase.php create mode 100644 search/Zend/Search/Lucene/Search/QueryEntry/Subquery.php create mode 100644 search/Zend/Search/Lucene/Search/QueryEntry/Term.php create mode 100644 search/Zend/Search/Lucene/Search/QueryLexer.php create mode 100644 search/Zend/Search/Lucene/Search/QueryParserContext.php create mode 100644 search/Zend/Search/Lucene/Search/QueryParserException.php create mode 100644 search/Zend/Search/Lucene/Search/Weight/Boolean.php create mode 100644 search/Zend/Search/Lucene/Search/Weight/Empty.php create mode 100644 search/Zend/Search/Lucene/Storage/File/Memory.php diff --git a/search/Zend/Search/Exception.php b/search/Zend/Search/Exception.php index a111cf6fcca..291cc43ed50 100644 --- a/search/Zend/Search/Exception.php +++ b/search/Zend/Search/Exception.php @@ -14,7 +14,7 @@ * * @category Zend * @package Zend_Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -22,13 +22,13 @@ /** * Framework base exception */ -require_once 'Zend/Exception.php'; +require_once $CFG->dirroot.'/search/Zend/Exception.php'; /** * @category Zend * @package Zend_Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Exception extends Zend_Exception diff --git a/search/Zend/Search/Lucene.php b/search/Zend/Search/Lucene.php index 3e33b7c1032..1f15c9a0ba1 100644 --- a/search/Zend/Search/Lucene.php +++ b/search/Zend/Search/Lucene.php @@ -14,53 +14,78 @@ * * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Exception */ -require_once 'Zend/Search/Lucene/Exception.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; /** Zend_Search_Lucene_Document */ -require_once 'Zend/Search/Lucene/Document.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Document.php'; + +/** Zend_Search_Lucene_Document_Html */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Document/Html.php'; /** Zend_Search_Lucene_Storage_Directory */ -require_once 'Zend/Search/Lucene/Storage/Directory/Filesystem.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/Directory/Filesystem.php'; + +/** Zend_Search_Lucene_Storage_File_Memory */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/File/Memory.php'; /** Zend_Search_Lucene_Index_Term */ -require_once 'Zend/Search/Lucene/Index/Term.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php'; /** Zend_Search_Lucene_Index_TermInfo */ -require_once 'Zend/Search/Lucene/Index/TermInfo.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/TermInfo.php'; /** Zend_Search_Lucene_Index_SegmentInfo */ -require_once 'Zend/Search/Lucene/Index/SegmentInfo.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php'; /** Zend_Search_Lucene_Index_FieldInfo */ -require_once 'Zend/Search/Lucene/Index/FieldInfo.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/FieldInfo.php'; /** Zend_Search_Lucene_Index_Writer */ -require_once 'Zend/Search/Lucene/Index/Writer.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Writer.php'; /** Zend_Search_Lucene_Search_QueryParser */ -require_once 'Zend/Search/Lucene/Search/QueryParser.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParser.php'; /** Zend_Search_Lucene_Search_QueryHit */ -require_once 'Zend/Search/Lucene/Search/QueryHit.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryHit.php'; /** Zend_Search_Lucene_Search_Similarity */ -require_once 'Zend/Search/Lucene/Search/Similarity.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Similarity.php'; + +/** Zend_Search_Lucene_Index_SegmentInfoPriorityQueue */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php'; + + +/** Zend_Search_Lucene_Interface */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Interface.php'; + +/** Zend_Search_Lucene_Proxy */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Proxy.php'; /** * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ -class Zend_Search_Lucene +class Zend_Search_Lucene implements Zend_Search_Lucene_Interface { + /** + * Default field name for search + * + * Null means search through all fields + * + * @var string + */ + private static $_defaultSearchField = null; + /** * File system adapter. * @@ -103,6 +128,51 @@ class Zend_Search_Lucene */ private $_hasChanges = false; + + /** + * Index lock object + * + * @var Zend_Search_Lucene_Storage_File + */ + private $_lock; + + /** + * Signal, that index is already closed, changes are fixed and resources are cleaned up + * + * @var boolean + */ + private $_closed = false; + + /** + * Number of references to the index object + * + * @var integer + */ + private $_refCount = 0; + + + /** + * Create index + * + * @param mixed $directory + * @return Zend_Search_Lucene_Interface + */ + public static function create($directory) + { + return new Zend_Search_Lucene_Proxy(new Zend_Search_Lucene($directory, true)); + } + + /** + * Open index + * + * @param mixed $directory + * @return Zend_Search_Lucene_Interface + */ + public static function open($directory) + { + return new Zend_Search_Lucene_Proxy(new Zend_Search_Lucene($directory, false)); + } + /** * Opens the index. * @@ -126,13 +196,32 @@ class Zend_Search_Lucene $this->_closeDirOnExit = true; } + + // Get a shared lock to the index + $this->_lock = $this->_directory->createFile('index.lock'); + + $this->_segmentInfos = array(); + if ($create) { - $this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory, true); + // Throw an exception if index is under processing now + if (!$this->_lock->lock(LOCK_EX, true)) { + throw new Zend_Search_Lucene_Exception('Can\'t create index. It\'s under processing now'); + } + + // Writer will create segments file for empty segments list + $this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory, $this->_segmentInfos, true); + + if (!$this->_lock->lock(LOCK_SH)) { + throw new Zend_Search_Lucene_Exception('Can\'t reduce lock level from Exclusive to Shared'); + } } else { + // Wait if index is under switching from one set of segments to another (Index_Writer::_updateSegments()) + if (!$this->_lock->lock(LOCK_SH)) { + throw new Zend_Search_Lucene_Exception('Can\'t obtain shared index lock'); + } $this->_writer = null; } - $this->_segmentInfos = array(); $segmentsFile = $this->_directory->getFileObject('segments'); @@ -143,9 +232,10 @@ class Zend_Search_Lucene } // read version - $segmentsFile->readLong(); + // $segmentsFile->readLong(); + $segmentsFile->readInt(); $segmentsFile->readInt(); - // read counter + // read segment name counter $segmentsFile->readInt(); $segments = $segmentsFile->readInt(); @@ -158,35 +248,83 @@ class Zend_Search_Lucene $segSize = $segmentsFile->readInt(); $this->_docCount += $segSize; - $this->_segmentInfos[$count] = + $this->_segmentInfos[] = new Zend_Search_Lucene_Index_SegmentInfo($segName, $segSize, $this->_directory); } } + /** + * Close current index and free resources + */ + private function _close() + { + if ($this->_closed) { + // index is already closed and resources are cleaned up + return; + } + + $this->commit(); + + // Free shared lock + $this->_lock->unlock(); + + if ($this->_closeDirOnExit) { + $this->_directory->close(); + } + + $this->_directory = null; + $this->_writer = null; + $this->_segmentInfos = null; + + $this->_closed = true; + } + + /** + * Add reference to the index object + * + * @internal + */ + public function addReference() + { + $this->_refCount++; + } + + /** + * Remove reference from the index object + * + * When reference count becomes zero, index is closed and resources are cleaned up + * + * @internal + */ + public function removeReference() + { + $this->_refCount--; + + if ($this->_refCount == 0) { + $this->_close(); + } + } /** * Object destructor */ public function __destruct() { - $this->commit(); - - if ($this->_closeDirOnExit) { - $this->_directory->close(); - } + $this->_close(); } /** * Returns an instance of Zend_Search_Lucene_Index_Writer for the index * + * @internal * @return Zend_Search_Lucene_Index_Writer */ public function getIndexWriter() { if (!$this->_writer instanceof Zend_Search_Lucene_Index_Writer) { - $this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory); + $this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory, $this->_segmentInfos); } return $this->_writer; @@ -205,7 +343,7 @@ class Zend_Search_Lucene /** - * Returns the total number of documents in this index. + * Returns the total number of documents in this index (including deleted documents). * * @return integer */ @@ -214,6 +352,192 @@ class Zend_Search_Lucene return $this->_docCount; } + /** + * Returns one greater than the largest possible document number. + * This may be used to, e.g., determine how big to allocate a structure which will have + * an element for every document number in an index. + * + * @return integer + */ + public function maxDoc() + { + return $this->count(); + } + + /** + * Returns the total number of non-deleted documents in this index. + * + * @return integer + */ + public function numDocs() + { + $numDocs = 0; + + foreach ($this->_segmentInfos as $segmentInfo) { + $numDocs += $segmentInfo->numDocs(); + } + + return $numDocs; + } + + /** + * Checks, that document is deleted + * + * @param integer $id + * @return boolean + * @throws Zend_Search_Lucene_Exception Exception is thrown if $id is out of the range + */ + public function isDeleted($id) + { + if ($id >= $this->_docCount) { + throw new Zend_Search_Lucene_Exception('Document id is out of the range.'); + } + + $segmentStartId = 0; + foreach ($this->_segmentInfos as $segmentInfo) { + if ($segmentStartId + $segmentInfo->count() > $id) { + break; + } + + $segmentStartId += $segmentInfo->count(); + } + + return $segmentInfo->isDeleted($id - $segmentStartId); + } + + /** + * Set default search field. + * + * Null means, that search is performed through all fields by default + * + * Default value is null + * + * @param string $fieldName + */ + public static function setDefaultSearchField($fieldName) + { + self::$_defaultSearchField = $fieldName; + } + + /** + * Get default search field. + * + * Null means, that search is performed through all fields by default + * + * @return string + */ + public static function getDefaultSearchField() + { + return self::$_defaultSearchField; + } + + /** + * Retrieve index maxBufferedDocs option + * + * maxBufferedDocs is a minimal number of documents required before + * the buffered in-memory documents are written into a new Segment + * + * Default value is 10 + * + * @return integer + */ + public function getMaxBufferedDocs() + { + return $this->getIndexWriter()->maxBufferedDocs; + } + + /** + * Set index maxBufferedDocs option + * + * maxBufferedDocs is a minimal number of documents required before + * the buffered in-memory documents are written into a new Segment + * + * Default value is 10 + * + * @param integer $maxBufferedDocs + */ + public function setMaxBufferedDocs($maxBufferedDocs) + { + $this->getIndexWriter()->maxBufferedDocs = $maxBufferedDocs; + } + + /** + * Retrieve index maxMergeDocs option + * + * maxMergeDocs is a largest number of documents ever merged by addDocument(). + * Small values (e.g., less than 10,000) are best for interactive indexing, + * as this limits the length of pauses while indexing to a few seconds. + * Larger values are best for batched indexing and speedier searches. + * + * Default value is PHP_INT_MAX + * + * @return integer + */ + public function getMaxMergeDocs() + { + return $this->getIndexWriter()->maxMergeDocs; + } + + /** + * Set index maxMergeDocs option + * + * maxMergeDocs is a largest number of documents ever merged by addDocument(). + * Small values (e.g., less than 10,000) are best for interactive indexing, + * as this limits the length of pauses while indexing to a few seconds. + * Larger values are best for batched indexing and speedier searches. + * + * Default value is PHP_INT_MAX + * + * @param integer $maxMergeDocs + */ + public function setMaxMergeDocs($maxMergeDocs) + { + $this->getIndexWriter()->maxMergeDocs = $maxMergeDocs; + } + + /** + * Retrieve index mergeFactor option + * + * mergeFactor determines how often segment indices are merged by addDocument(). + * With smaller values, less RAM is used while indexing, + * and searches on unoptimized indices are faster, + * but indexing speed is slower. + * With larger values, more RAM is used during indexing, + * and while searches on unoptimized indices are slower, + * indexing is faster. + * Thus larger values (> 10) are best for batch index creation, + * and smaller values (< 10) for indices that are interactively maintained. + * + * Default value is 10 + * + * @return integer + */ + public function getMergeFactor() + { + return $this->getIndexWriter()->mergeFactor; + } + + /** + * Set index mergeFactor option + * + * mergeFactor determines how often segment indices are merged by addDocument(). + * With smaller values, less RAM is used while indexing, + * and searches on unoptimized indices are faster, + * but indexing speed is slower. + * With larger values, more RAM is used during indexing, + * and while searches on unoptimized indices are slower, + * indexing is faster. + * Thus larger values (> 10) are best for batch index creation, + * and smaller values (< 10) for indices that are interactively maintained. + * + * Default value is 10 + * + * @param integer $maxMergeDocs + */ + public function setMergeFactor($mergeFactor) + { + $this->getIndexWriter()->mergeFactor = $mergeFactor; + } /** * Performs a query against the index and returns an array @@ -221,7 +545,8 @@ class Zend_Search_Lucene * Input is a string or Zend_Search_Lucene_Search_Query. * * @param mixed $query - * @return array ZSearchHit + * @return array Zend_Search_Lucene_Search_QueryHit + * @throws Zend_Search_Lucene_Exception */ public function find($query) { @@ -235,22 +560,115 @@ class Zend_Search_Lucene $this->commit(); - $hits = array(); + $hits = array(); $scores = array(); + $ids = array(); - $docNum = $this->count(); - for( $count=0; $count < $docNum; $count++ ) { - $docScore = $query->score( $count, $this); + $query = $query->rewrite($this)->optimize($this); + + $query->execute($this); + + $topScore = 0; + + foreach ($query->matchedDocs() as $id => $num) { + $docScore = $query->score($id, $this); if( $docScore != 0 ) { $hit = new Zend_Search_Lucene_Search_QueryHit($this); - $hit->id = $count; + $hit->id = $id; $hit->score = $docScore; - $hits[] = $hit; + $hits[] = $hit; + $ids[] = $id; $scores[] = $docScore; + + if ($docScore > $topScore) { + $topScore = $docScore; + } } } - array_multisort($scores, SORT_DESC, SORT_REGULAR, $hits); + + if (count($hits) == 0) { + // skip sorting, which may cause a error on empty index + return array(); + } + + if ($topScore > 1) { + foreach ($hits as $hit) { + $hit->score /= $topScore; + } + } + + if (func_num_args() == 1) { + // sort by scores + array_multisort($scores, SORT_DESC, SORT_NUMERIC, + $ids, SORT_ASC, SORT_NUMERIC, + $hits); + } else { + // sort by given field names + + $argList = func_get_args(); + $fieldNames = $this->getFieldNames(); + $sortArgs = array(); + + for ($count = 1; $count < count($argList); $count++) { + $fieldName = $argList[$count]; + + if (!is_string($fieldName)) { + throw new Zend_Search_Lucene_Exception('Field name must be a string.'); + } + + if (!in_array($fieldName, $fieldNames)) { + throw new Zend_Search_Lucene_Exception('Wrong field name.'); + } + + $valuesArray = array(); + foreach ($hits as $hit) { + try { + $value = $hit->getDocument()->getFieldValue($fieldName); + } catch (Zend_Search_Lucene_Exception $e) { + if (strpos($e->getMessage(), 'not found') === false) { + throw $e; + } else { + $value = null; + } + } + + $valuesArray[] = $value; + } + + $sortArgs[] = $valuesArray; + + if ($count + 1 < count($argList) && is_integer($argList[$count+1])) { + $count++; + $sortArgs[] = $argList[$count]; + + if ($count + 1 < count($argList) && is_integer($argList[$count+1])) { + $count++; + $sortArgs[] = $argList[$count]; + } else { + if ($argList[$count] == SORT_ASC || $argList[$count] == SORT_DESC) { + $sortArgs[] = SORT_REGULAR; + } else { + $sortArgs[] = SORT_ASC; + } + } + } else { + $sortArgs[] = SORT_ASC; + $sortArgs[] = SORT_REGULAR; + } + } + + // Sort by id's if values are equal + $sortArgs[] = $ids; + $sortArgs[] = SORT_ASC; + $sortArgs[] = SORT_NUMERIC; + + // Array to be sorted + $sortArgs[] = &$hits; + + // Do sort + call_user_func_array('array_multisort', $sortArgs); + } return $hits; } @@ -290,41 +708,45 @@ class Zend_Search_Lucene throw new Zend_Search_Lucene_Exception('Document id is out of the range.'); } - $segCount = 0; - $nextSegmentStartId = $this->_segmentInfos[ 0 ]->count(); - while( $nextSegmentStartId <= $id ) { - $segCount++; - $nextSegmentStartId += $this->_segmentInfos[ $segCount ]->count(); - } - $segmentStartId = $nextSegmentStartId - $this->_segmentInfos[ $segCount ]->count(); + $segmentStartId = 0; + foreach ($this->_segmentInfos as $segmentInfo) { + if ($segmentStartId + $segmentInfo->count() > $id) { + break; + } - $fdxFile = $this->_segmentInfos[ $segCount ]->openCompoundFile('.fdx'); + $segmentStartId += $segmentInfo->count(); + } + + $fdxFile = $segmentInfo->openCompoundFile('.fdx'); $fdxFile->seek( ($id-$segmentStartId)*8, SEEK_CUR ); $fieldValuesPosition = $fdxFile->readLong(); - $fdtFile = $this->_segmentInfos[ $segCount ]->openCompoundFile('.fdt'); - $fdtFile->seek( $fieldValuesPosition, SEEK_CUR ); + $fdtFile = $segmentInfo->openCompoundFile('.fdt'); + $fdtFile->seek($fieldValuesPosition, SEEK_CUR); $fieldCount = $fdtFile->readVInt(); $doc = new Zend_Search_Lucene_Document(); - for( $count = 0; $count < $fieldCount; $count++ ) { + for ($count = 0; $count < $fieldCount; $count++) { $fieldNum = $fdtFile->readVInt(); $bits = $fdtFile->readByte(); - $fieldInfo = $this->_segmentInfos[ $segCount ]->getField($fieldNum); + $fieldInfo = $segmentInfo->getField($fieldNum); - if( !($bits & 2) ) { // Text data + if (!($bits & 2)) { // Text data $field = new Zend_Search_Lucene_Field($fieldInfo->name, $fdtFile->readString(), + 'UTF-8', true, $fieldInfo->isIndexed, $bits & 1 ); - } else { + } else { // Binary data $field = new Zend_Search_Lucene_Field($fieldInfo->name, $fdtFile->readBinary(), + '', true, $fieldInfo->isIndexed, - $bits & 1 ); + $bits & 1, + true ); } $doc->addField($field); @@ -335,7 +757,26 @@ class Zend_Search_Lucene /** - * Returns an array of all the documents which contain term. + * Returns true if index contain documents with specified term. + * + * Is used for query optimization. + * + * @param Zend_Search_Lucene_Index_Term $term + * @return boolean + */ + public function hasTerm(Zend_Search_Lucene_Index_Term $term) + { + foreach ($this->_segmentInfos as $segInfo) { + if ($segInfo->getTermInfo($term) instanceof Zend_Search_Lucene_Index_TermInfo) { + return true; + } + } + + return false; + } + + /** + * Returns IDs of all the documents containing term. * * @param Zend_Search_Lucene_Index_Term $term * @return array @@ -376,9 +817,29 @@ class Zend_Search_Lucene } + /** + * Returns an array of all term freqs. + * Result array structure: array(docId => freq, ...) + * + * @param Zend_Search_Lucene_Index_Term $term + * @return integer + */ + public function termFreqs(Zend_Search_Lucene_Index_Term $term) + { + $result = array(); + $segmentStartDocId = 0; + foreach ($this->_segmentInfos as $segmentInfo) { + $result += $segmentInfo->termFreqs($term, $segmentStartDocId); + + $segmentStartDocId += $segmentInfo->count(); + } + + return $result; + } + /** * Returns an array of all term positions in the documents. - * Return array structure: array( docId => array( pos1, pos2, ...), ...) + * Result array structure: array(docId => array(pos1, pos2, ...), ...) * * @param Zend_Search_Lucene_Index_Term $term * @return array @@ -387,45 +848,10 @@ class Zend_Search_Lucene { $result = array(); $segmentStartDocId = 0; - foreach( $this->_segmentInfos as $segInfo ) { - $termInfo = $segInfo->getTermInfo($term); + foreach ($this->_segmentInfos as $segmentInfo) { + $result += $segmentInfo->termPositions($term, $segmentStartDocId); - if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) { - $segmentStartDocId += $segInfo->count(); - continue; - } - - $frqFile = $segInfo->openCompoundFile('.frq'); - $frqFile->seek($termInfo->freqPointer,SEEK_CUR); - $freqs = array(); - $docId = 0; - - for( $count = 0; $count < $termInfo->docFreq; $count++ ) { - $docDelta = $frqFile->readVInt(); - if( $docDelta % 2 == 1 ) { - $docId += ($docDelta-1)/2; - $freqs[ $docId ] = 1; - } else { - $docId += $docDelta/2; - $freqs[ $docId ] = $frqFile->readVInt(); - } - } - - $prxFile = $segInfo->openCompoundFile('.prx'); - $prxFile->seek($termInfo->proxPointer,SEEK_CUR); - foreach ($freqs as $docId => $freq) { - $termPosition = 0; - $positions = array(); - - for ($count = 0; $count < $freq; $count++ ) { - $termPosition += $prxFile->readVInt(); - $positions[] = $termPosition; - } - - $result[ $segmentStartDocId + $docId ] = $positions; - } - - $segmentStartDocId += $segInfo->count(); + $segmentStartDocId += $segmentInfo->count(); } return $result; @@ -468,9 +894,9 @@ class Zend_Search_Lucene * * @param integer $id * @param string $fieldName - * @return Zend_Search_Lucene_Document + * @return float */ - public function norm( $id, $fieldName ) + public function norm($id, $fieldName) { if ($id >= $this->_docCount) { return null; @@ -527,16 +953,17 @@ class Zend_Search_Lucene throw new Zend_Search_Lucene_Exception('Document id is out of the range.'); } - $segCount = 0; - $nextSegmentStartId = $this->_segmentInfos[ 0 ]->count(); - while( $nextSegmentStartId <= $id ) { - $segCount++; - $nextSegmentStartId += $this->_segmentInfos[ $segCount ]->count(); + $segmentStartId = 0; + foreach ($this->_segmentInfos as $segmentInfo) { + if ($segmentStartId + $segmentInfo->count() > $id) { + break; + } + + $segmentStartId += $segmentInfo->count(); } + $segmentInfo->delete($id - $segmentStartId); $this->_hasChanges = true; - $segmentStartId = $nextSegmentStartId - $this->_segmentInfos[ $segCount ]->count(); - $this->_segmentInfos[ $segCount ]->delete($id - $segmentStartId); } @@ -548,18 +975,26 @@ class Zend_Search_Lucene */ public function addDocument(Zend_Search_Lucene_Document $document) { - if (!$this->_writer instanceof Zend_Search_Lucene_Index_Writer) { - $this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory); - } - - $this->_writer->addDocument($document); + $this->getIndexWriter()->addDocument($document); + $this->_docCount++; } + /** + * Update document counter + */ + private function _updateDocCount() + { + $this->_docCount = 0; + foreach ($this->_segmentInfos as $segInfo) { + $this->_docCount += $segInfo->count(); + } + } + /** * Commit changes resulting from delete() or undeleteAll() operations. * - * @todo delete() and undeleteAll processing. + * @todo undeleteAll processing. */ public function commit() { @@ -572,38 +1007,73 @@ class Zend_Search_Lucene } if ($this->_writer !== null) { - foreach ($this->_writer->commit() as $segmentName => $segmentInfo) { - if ($segmentInfo !== null) { - $this->_segmentInfos[] = $segmentInfo; - $this->_docCount += $segmentInfo->count(); - } else { - foreach ($this->_segmentInfos as $segId => $segInfo) { - if ($segInfo->getName() == $segmentName) { - unset($this->_segmentInfos[$segId]); - } - } - } + $this->_writer->commit(); + + $this->_updateDocCount(); + } + } + + + /** + * Optimize index. + * + * Merges all segments into one + */ + public function optimize() + { + // Commit changes if any changes have been made + $this->commit(); + + if (count($this->_segmentInfos) > 1 || $this->hasDeletions()) { + $this->getIndexWriter()->optimize(); + $this->_updateDocCount(); + } + } + + + /** + * Returns an array of all terms in this index. + * + * @return array + */ + public function terms() + { + $result = array(); + + $segmentInfoQueue = new Zend_Search_Lucene_Index_SegmentInfoPriorityQueue(); + + foreach ($this->_segmentInfos as $segmentInfo) { + $segmentInfo->reset(); + + // Skip "empty" segments + if ($segmentInfo->currentTerm() !== null) { + $segmentInfoQueue->put($segmentInfo); } } + + while (($segmentInfo = $segmentInfoQueue->pop()) !== null) { + if ($segmentInfoQueue->top() === null || + $segmentInfoQueue->top()->currentTerm()->key() != + $segmentInfo->currentTerm()->key()) { + // We got new term + $result[] = $segmentInfo->currentTerm(); + } + + $segmentInfo->nextTerm(); + // check, if segment dictionary is finished + if ($segmentInfo->currentTerm() !== null) { + // Put segment back into the priority queue + $segmentInfoQueue->put($segmentInfo); + } + } + + return $result; } /************************************************************************* @todo UNIMPLEMENTED *************************************************************************/ - - /** - * Returns an array of all terms in this index. - * - * @todo Implementation - * @return array - */ - public function terms() - { - return array(); - } - - /** * Undeletes all documents currently marked as deleted in this index. * @@ -611,4 +1081,4 @@ class Zend_Search_Lucene */ public function undeleteAll() {} -} \ No newline at end of file +} diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer.php b/search/Zend/Search/Lucene/Analysis/Analyzer.php index febf88e6140..7a29c763d90 100644 --- a/search/Zend/Search/Lucene/Analysis/Analyzer.php +++ b/search/Zend/Search/Lucene/Analysis/Analyzer.php @@ -15,20 +15,37 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_Token */ -require_once 'Zend/Search/Lucene/Analysis/Token.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Token.php'; + +/** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8 */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php'; + +/** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php'; /** Zend_Search_Lucene_Analysis_Analyzer_Common_Text */ -require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php'; /** Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive */ -require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php'; +/** Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php'; + +/** Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php'; + +/** Zend_Search_Lucene_Analysis_TokenFilter_StopWords */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php'; + +/** Zend_Search_Lucene_Analysis_TokenFilter_ShortWords */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php'; /** @@ -44,7 +61,7 @@ require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.p * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -55,16 +72,73 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer * * @var Zend_Search_Lucene_Analysis_Analyzer */ - static private $_defaultImpl; + private static $_defaultImpl; + + /** + * Input string + * + * @var string + */ + protected $_input = null; + + /** + * Input string encoding + * + * @var string + */ + protected $_encoding = ''; /** * Tokenize text to a terms * Returns array of Zend_Search_Lucene_Analysis_Token objects * + * Tokens are returned in UTF-8 (internal Zend_Search_Lucene encoding) + * * @param string $data * @return array */ - abstract public function tokenize($data); + public function tokenize($data, $encoding = 'UTF-8') + { + $this->setInput($data, $encoding); + $tokenList = array(); + while (($nextToken = $this->nextToken()) !== null) { + $tokenList[] = $nextToken; + } + + return $tokenList; + } + + + /** + * Tokenization stream API + * Set input + * + * @param string $data + */ + public function setInput($data, $encoding = '') + { + $this->_input = $data; + $this->_encoding = $encoding; + $this->reset(); + } + + /** + * Reset token stream + */ + abstract public function reset(); + + /** + * Tokenization stream API + * Get next token + * Returns null at the end of stream + * + * Tokens are returned in UTF-8 (internal Zend_Search_Lucene encoding) + * + * @return Zend_Search_Lucene_Analysis_Token|null + */ + abstract public function nextToken(); + + /** @@ -72,7 +146,7 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer * * @param Zend_Search_Lucene_Analysis_Analyzer $similarity */ - static public function setDefault(Zend_Search_Lucene_Analysis_Analyzer $analyzer) + public static function setDefault(Zend_Search_Lucene_Analysis_Analyzer $analyzer) { self::$_defaultImpl = $analyzer; } @@ -83,14 +157,13 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer * * @return Zend_Search_Lucene_Analysis_Analyzer */ - static public function getDefault() + public static function getDefault() { if (!self::$_defaultImpl instanceof Zend_Search_Lucene_Analysis_Analyzer) { - self::$_defaultImpl = new Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive(); + self::$_defaultImpl = new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8(); } return self::$_defaultImpl; } - } diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php b/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php index 2ad8a0516b6..c518e930717 100644 --- a/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php +++ b/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php @@ -15,13 +15,13 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_Analyzer */ -require_once 'Zend/Search/Lucene/Analysis/Analyzer.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php'; /** @@ -34,7 +34,7 @@ require_once 'Zend/Search/Lucene/Analysis/Analyzer.php'; * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_Analysis_Analyzer_Common extends Zend_Search_Lucene_Analysis_Analyzer @@ -58,7 +58,7 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer_Common extends Zend_Search_L } /** - * Apply filters to the token. + * Apply filters to the token. Can return null when the token was removed. * * @param Zend_Search_Lucene_Analysis_Token $token * @return Zend_Search_Lucene_Analysis_Token @@ -67,6 +67,11 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer_Common extends Zend_Search_L { foreach ($this->_filters as $filter) { $token = $filter->normalize($token); + + // resulting token can be null if the filter removed it + if (is_null($token)) { + return null; + } } return $token; diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php index 6f6f0dd9366..d084ebc4c5f 100644 --- a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php +++ b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php @@ -15,64 +15,79 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_Analyzer_Common */ -require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Analysis_Analyzer_Common_Text extends Zend_Search_Lucene_Analysis_Analyzer_Common { /** - * Tokenize text to a terms - * Returns array of Zend_Search_Lucene_Analysis_Token objects + * Current position in a stream * - * @param string $data - * @return array + * @var integer */ - public function tokenize($data) + private $_position; + + /** + * Reset token stream + */ + public function reset() { - $tokenStream = array(); + $this->_position = 0; - $position = 0; - while ($position < strlen($data)) { - // skip white space - while ($position < strlen($data) && !ctype_alpha( $data{$position} )) { - $position++; - } - - $termStartPosition = $position; - - // read token - while ($position < strlen($data) && ctype_alpha( $data{$position} )) { - $position++; - } - - // Empty token, end of stream. - if ($position == $termStartPosition) { - break; - } - - $token = new Zend_Search_Lucene_Analysis_Token(substr($data, - $termStartPosition, - $position-$termStartPosition), - $termStartPosition, - $position); - $tokenStream[] = $this->normalize($token); + if ($this->_input === null) { + return; } - return $tokenStream; + // convert input into ascii + $this->_input = iconv($this->_encoding, 'ASCII//TRANSLIT', $this->_input); + $this->_encoding = 'ASCII'; + } + + /** + * Tokenization stream API + * Get next token + * Returns null at the end of stream + * + * @return Zend_Search_Lucene_Analysis_Token|null + */ + public function nextToken() + { + if ($this->_input === null) { + return null; + } + + + do { + if (! preg_match('/[a-zA-Z]+/', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_position)) { + // It covers both cases a) there are no matches (preg_match(...) === 0) + // b) error occured (preg_match(...) === FALSE) + return null; + } + + $str = $match[0][0]; + $pos = $match[0][1]; + $endpos = $pos + strlen($str); + + $this->_position = $endpos; + + $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($str, $pos, $endpos)); + } while ($token === null); // try again if token is skipped + + return $token; } } diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php index e5fc3726280..d9f786a8703 100644 --- a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php +++ b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php @@ -15,23 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_Analyzer_Common_Text */ -require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php'; /** Zend_Search_Lucene_Analysis_TokenFilter_LowerCase */ -require_once 'Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php new file mode 100644 index 00000000000..d68b594a077 --- /dev/null +++ b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php @@ -0,0 +1,92 @@ +dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Analysis + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ + +class Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum extends Zend_Search_Lucene_Analysis_Analyzer_Common +{ + /** + * Current position in a stream + * + * @var integer + */ + private $_position; + + /** + * Reset token stream + */ + public function reset() + { + $this->_position = 0; + + if ($this->_input === null) { + return; + } + + // convert input into ascii + $this->_input = iconv($this->_encoding, 'ASCII//TRANSLIT', $this->_input); + $this->_encoding = 'ASCII'; + } + + /** + * Tokenization stream API + * Get next token + * Returns null at the end of stream + * + * @return Zend_Search_Lucene_Analysis_Token|null + */ + public function nextToken() + { + if ($this->_input === null) { + return null; + } + + do { + if (! preg_match('/[a-zA-Z0-9]+/', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_position)) { + // It covers both cases a) there are no matches (preg_match(...) === 0) + // b) error occured (preg_match(...) === FALSE) + return null; + } + + $str = $match[0][0]; + $pos = $match[0][1]; + $endpos = $pos + strlen($str); + + $this->_position = $endpos; + + $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($str, $pos, $endpos)); + } while ($token === null); // try again if token is skipped + + return $token; + } +} + diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php new file mode 100644 index 00000000000..6eab4372291 --- /dev/null +++ b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php @@ -0,0 +1,46 @@ +dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php'; + +/** Zend_Search_Lucene_Analysis_TokenFilter_LowerCase */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Analysis + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ + + +class Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive extends Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum +{ + public function __construct() + { + $this->addFilter(new Zend_Search_Lucene_Analysis_TokenFilter_LowerCase()); + } +} + diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php new file mode 100644 index 00000000000..674a3d9e643 --- /dev/null +++ b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php @@ -0,0 +1,169 @@ +dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Analysis + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ + +class Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8 extends Zend_Search_Lucene_Analysis_Analyzer_Common +{ + /** + * Current char position in an UTF-8 stream + * + * @var integer + */ + private $_position; + + /** + * Current binary position in an UTF-8 stream + * + * @var integer + */ + private $_bytePosition; + + /** + * Stream length + * + * @var integer + */ + private $_streamLength; + + /** + * Reset token stream + */ + public function reset() + { + $this->_position = 0; + $this->_bytePosition = 0; + + // convert input into UTF-8 + if (strcasecmp($this->_encoding, 'utf8' ) != 0 && + strcasecmp($this->_encoding, 'utf-8') != 0 ) { + $this->_input = iconv($this->_encoding, 'UTF-8', $this->_input); + $this->_encoding = 'UTF-8'; + } + + // Get UTF-8 string length. + // It also checks if it's a correct utf-8 string + $this->_streamLength = iconv_strlen($this->_input, 'UTF-8'); + } + + /** + * Check, that character is a letter + * + * @param string $char + * @return boolean + */ + private static function _isAlpha($char) + { + if (strlen($char) > 1) { + // It's an UTF-8 character + return true; + } + + return ctype_alpha($char); + } + + /** + * Get next UTF-8 char + * + * @param string $char + * @return boolean + */ + private function _nextChar() + { + $char = $this->_input[$this->_bytePosition++]; + + if (( ord($char) & 0xC0 ) == 0xC0) { + $addBytes = 1; + if (ord($char) & 0x20 ) { + $addBytes++; + if (ord($char) & 0x10 ) { + $addBytes++; + } + } + $char .= substr($this->_input, $this->_bytePosition, $addBytes); + $this->_bytePosition += $addBytes; + } + + $this->_position++; + + return $char; + } + + /** + * Tokenization stream API + * Get next token + * Returns null at the end of stream + * + * @return Zend_Search_Lucene_Analysis_Token|null + */ + public function nextToken() + { + if ($this->_input === null) { + return null; + } + + while ($this->_position < $this->_streamLength) { + // skip white space + while ($this->_position < $this->_streamLength && + !self::_isAlpha($char = $this->_nextChar())) { + $char = ''; + } + + $termStartPosition = $this->_position - 1; + $termText = $char; + + // read token + while ($this->_position < $this->_streamLength && + self::_isAlpha($char = $this->_nextChar())) { + $termText .= $char; + } + + // Empty token, end of stream. + if ($termText == '') { + return null; + } + + $token = new Zend_Search_Lucene_Analysis_Token( + $termText, + $termStartPosition, + $this->_position - 1); + $token = $this->normalize($token); + if ($token !== null) { + return $token; + } + // Continue if token is skipped + } + + return null; + } +} + diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php new file mode 100644 index 00000000000..982b55418fd --- /dev/null +++ b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php @@ -0,0 +1,169 @@ +dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Analysis + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ + +class Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num extends Zend_Search_Lucene_Analysis_Analyzer_Common +{ + /** + * Current char position in an UTF-8 stream + * + * @var integer + */ + private $_position; + + /** + * Current binary position in an UTF-8 stream + * + * @var integer + */ + private $_bytePosition; + + /** + * Stream length + * + * @var integer + */ + private $_streamLength; + + /** + * Reset token stream + */ + public function reset() + { + $this->_position = 0; + $this->_bytePosition = 0; + + // convert input into UTF-8 + if (strcasecmp($this->_encoding, 'utf8' ) != 0 && + strcasecmp($this->_encoding, 'utf-8') != 0 ) { + $this->_input = iconv($this->_encoding, 'UTF-8', $this->_input); + $this->_encoding = 'UTF-8'; + } + + // Get UTF-8 string length. + // It also checks if it's a correct utf-8 string + $this->_streamLength = iconv_strlen($this->_input, 'UTF-8'); + } + + /** + * Check, that character is a letter + * + * @param string $char + * @return boolean + */ + private static function _isAlNum($char) + { + if (strlen($char) > 1) { + // It's an UTF-8 character + return true; + } + + return ctype_alnum($char); + } + + /** + * Get next UTF-8 char + * + * @param string $char + * @return boolean + */ + private function _nextChar() + { + $char = $this->_input[$this->_bytePosition++]; + + if (( ord($char) & 0xC0 ) == 0xC0) { + $addBytes = 1; + if (ord($char) & 0x20 ) { + $addBytes++; + if (ord($char) & 0x10 ) { + $addBytes++; + } + } + $char .= substr($this->_input, $this->_bytePosition, $addBytes); + $this->_bytePosition += $addBytes; + } + + $this->_position++; + + return $char; + } + + /** + * Tokenization stream API + * Get next token + * Returns null at the end of stream + * + * @return Zend_Search_Lucene_Analysis_Token|null + */ + public function nextToken() + { + if ($this->_input === null) { + return null; + } + + while ($this->_position < $this->_streamLength) { + // skip white space + while ($this->_position < $this->_streamLength && + !self::_isAlNum($char = $this->_nextChar())) { + $char = ''; + } + + $termStartPosition = $this->_position - 1; + $termText = $char; + + // read token + while ($this->_position < $this->_streamLength && + self::_isAlNum($char = $this->_nextChar())) { + $termText .= $char; + } + + // Empty token, end of stream. + if ($termText == '') { + return null; + } + + $token = new Zend_Search_Lucene_Analysis_Token( + $termText, + $termStartPosition, + $this->_position - 1); + $token = $this->normalize($token); + if ($token !== null) { + return $token; + } + // Continue if token is skipped + } + + return null; + } +} + diff --git a/search/Zend/Search/Lucene/Analysis/Token.php b/search/Zend/Search/Lucene/Analysis/Token.php index f2e9ee7cad9..91586b01f72 100644 --- a/search/Zend/Search/Lucene/Analysis/Token.php +++ b/search/Zend/Search/Lucene/Analysis/Token.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -24,7 +24,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Analysis_Token @@ -50,13 +50,6 @@ class Zend_Search_Lucene_Analysis_Token */ private $_endOffset; - /** - * Lexical type. - * - * @var string - */ - private $_type; - /** * The position of this token relative to the previous Token. * @@ -90,12 +83,11 @@ class Zend_Search_Lucene_Analysis_Token * @param integer $end * @param string $type */ - public function __construct($text, $start, $end, $type = 'word' ) + public function __construct($text, $start, $end) { $this->_termText = $text; $this->_startOffset = $start; $this->_endOffset = $end; - $this->_type = $type; $this->_positionIncrement = 1; } @@ -157,15 +149,5 @@ class Zend_Search_Lucene_Analysis_Token { return $this->_endOffset; } - - /** - * Returns this Token's lexical type. Defaults to 'word'. - * - * @return string - */ - public function getType() - { - return $this->_type; - } } diff --git a/search/Zend/Search/Lucene/Analysis/TokenFilter.php b/search/Zend/Search/Lucene/Analysis/TokenFilter.php index a363aa1c1c2..4d559a93b7f 100644 --- a/search/Zend/Search/Lucene/Analysis/TokenFilter.php +++ b/search/Zend/Search/Lucene/Analysis/TokenFilter.php @@ -15,13 +15,13 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_Token */ -require_once 'Zend/Search/Lucene/Analysis/Token.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Token.php'; /** @@ -30,7 +30,7 @@ require_once 'Zend/Search/Lucene/Analysis/Token.php'; * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ diff --git a/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php b/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php index 5ea1edf8321..01d25c67452 100644 --- a/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php +++ b/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php @@ -15,13 +15,13 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_TokenFilter */ -require_once 'Zend/Search/Lucene/Analysis/TokenFilter.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter.php'; /** @@ -30,7 +30,7 @@ require_once 'Zend/Search/Lucene/Analysis/TokenFilter.php'; * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -44,10 +44,10 @@ class Zend_Search_Lucene_Analysis_TokenFilter_LowerCase extends Zend_Search_Luce */ public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken) { - $newToken = new Zend_Search_Lucene_Analysis_Token(strtolower( $srcToken->getTermText() ), + $newToken = new Zend_Search_Lucene_Analysis_Token( + strtolower( $srcToken->getTermText() ), $srcToken->getStartOffset(), - $srcToken->getEndOffset(), - $srcToken->getType()); + $srcToken->getEndOffset()); $newToken->setPositionIncrement($srcToken->getPositionIncrement()); diff --git a/search/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php b/search/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php new file mode 100644 index 00000000000..83abfb22c3f --- /dev/null +++ b/search/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php @@ -0,0 +1,68 @@ +dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter.php'; + + +/** + * Token filter that removes short words. What is short word can be configured with constructor. + * + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Analysis + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ + +class Zend_Search_Lucene_Analysis_TokenFilter_ShortWords extends Zend_Search_Lucene_Analysis_TokenFilter +{ + /** + * Minimum allowed term length + * @var integer + */ + private $length; + + /** + * Constructs new instance of this filter. + * + * @param integer $short minimum allowed length of term which passes this filter (default 2) + */ + public function __construct($length = 2) { + $this->length = $length; + } + + /** + * Normalize Token or remove it (if null is returned) + * + * @param Zend_Search_Lucene_Analysis_Token $srcToken + * @return Zend_Search_Lucene_Analysis_Token + */ + public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken) { + if (strlen($srcToken->getTermText()) < $this->length) { + return null; + } else { + return $srcToken; + } + } +} + diff --git a/search/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php b/search/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php new file mode 100644 index 00000000000..f85d5d55d8e --- /dev/null +++ b/search/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php @@ -0,0 +1,101 @@ +dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Exception.php'; + + +/** + * Token filter that removes stop words. These words must be provided as array (set), example: + * $stopwords = array('the' => 1, 'an' => '1'); + * + * We do recommend to provide all words in lowercase and concatenate this class after the lowercase filter. + * + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Analysis + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ + +class Zend_Search_Lucene_Analysis_TokenFilter_StopWords extends Zend_Search_Lucene_Analysis_TokenFilter +{ + /** + * Minimum allowed term length + * @var array + */ + private $_stopSet; + + /** + * Constructs new instance of this filter. + * + * @param array $stopwords array (set) of words that will be filtered out + */ + public function __construct($stopwords = array()) { + $this->_stopSet = array_flip($stopwords); + } + + /** + * Normalize Token or remove it (if null is returned) + * + * @param Zend_Search_Lucene_Analysis_Token $srcToken + * @return Zend_Search_Lucene_Analysis_Token + */ + public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken) { + if (array_key_exists($srcToken->getTermText(), $this->_stopSet)) { + $t = $srcToken->getTermText(); + return null; + } else { + return $srcToken; + } + } + + /** + * Fills stopwords set from a text file. Each line contains one stopword, lines with '#' in the first + * column are ignored (as comments). + * + * You can call this method one or more times. New stopwords are always added to current set. + * + * @param string $filepath full path for text file with stopwords + * @throws Zend_Search_Exception When the file doesn`t exists or is not readable. + */ + public function loadFromFile($filepath = null) { + if (! $filepath || ! file_exists($filepath)) { + throw new Zend_Search_Exception('You have to provide valid file path'); + } + $fd = fopen($filepath, "r"); + if (! $fd) { + throw new Zend_Search_Exception('Cannot open file ' . $filepath); + } + while (!feof ($fd)) { + $buffer = trim(fgets($fd)); + if (strlen($buffer) > 0 && $buffer[0] != '#') { + $this->_stopSet[$buffer] = 1; + } + } + if (!fclose($fd)) { + throw new Zend_Search_Exception('Cannot close file ' . $filepath); + } + } +} + diff --git a/search/Zend/Search/Lucene/Document.php b/search/Zend/Search/Lucene/Document.php index 48e48cf17a8..6309719568d 100644 --- a/search/Zend/Search/Lucene/Document.php +++ b/search/Zend/Search/Lucene/Document.php @@ -15,13 +15,13 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Document - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Field */ -require_once 'Zend/Search/Lucene/Field.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Field.php'; /** @@ -30,7 +30,7 @@ require_once 'Zend/Search/Lucene/Field.php'; * @category Zend * @package Zend_Search_Lucene * @subpackage Document - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Document @@ -90,9 +90,9 @@ class Zend_Search_Lucene_Document */ public function getField($fieldName) { - if (!array_key_exists($fieldName, $this->_fields)) { - throw new Zend_Search_Lucene_Exception("Field name \"$fieldName\" not found in document."); - } + if (!array_key_exists($fieldName, $this->_fields)) { + throw new Zend_Search_Lucene_Exception("Field name \"$fieldName\" not found in document."); + } return $this->_fields[$fieldName]; } @@ -105,7 +105,17 @@ class Zend_Search_Lucene_Document */ public function getFieldValue($fieldName) { - return $this->getField($fieldName)->stringValue; + return $this->getField($fieldName)->value; } + /** + * Returns the string value of a named field in UTF-8 encoding. + * + * @see __get() + * @return string + */ + public function getFieldUtf8Value($fieldName) + { + return $this->getField($fieldName)->getUtf8Value(); + } } diff --git a/search/Zend/Search/Lucene/Document/Html.php b/search/Zend/Search/Lucene/Document/Html.php new file mode 100644 index 00000000000..c10c8236c5c --- /dev/null +++ b/search/Zend/Search/Lucene/Document/Html.php @@ -0,0 +1,310 @@ +dirroot.'/search/Zend/Search/Lucene/Document.php'; + + +/** + * HTML document. + * + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Document + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Document_Html extends Zend_Search_Lucene_Document +{ + /** + * List of document links + * + * @var array + */ + private $_links = array(); + + /** + * List of document header links + * + * @var array + */ + private $_headerLinks = array(); + + /** + * Stored DOM representation + * + * @var DOMDocument + */ + private $_doc; + + /** + * Object constructor + * + * @param string $data + * @param boolean $isFile + * @param boolean $storeContent + */ + private function __construct($data, $isFile, $storeContent) + { + $this->_doc = new DOMDocument(); + $this->_doc->substituteEntities = true; + + if ($isFile) { + @$this->_doc->loadHTMLFile($data); + } else{ + @$this->_doc->loadHTML($data); + } + + $xpath = new DOMXPath($this->_doc); + + $docTitle = ''; + $titleNodes = $xpath->query('/html/head/title'); + foreach ($titleNodes as $titleNode) { + // title should always have only one entry, but we process all nodeset entries + $docTitle .= $titleNode->nodeValue . ' '; + } + $this->addField(Zend_Search_Lucene_Field::Text('title', $docTitle, $this->_doc->actualEncoding)); + + $metaNodes = $xpath->query('/html/head/meta[@name]'); + foreach ($metaNodes as $metaNode) { + $this->addField(Zend_Search_Lucene_Field::Text($metaNode->getAttribute('name'), + $metaNode->getAttribute('content'), + $this->_doc->actualEncoding)); + } + + $docBody = ''; + $bodyNodes = $xpath->query('/html/body'); + foreach ($bodyNodes as $bodyNode) { + // body should always have only one entry, but we process all nodeset entries + $this->_retrieveNodeText($bodyNode, $docBody); + } + if ($storeContent) { + $this->addField(Zend_Search_Lucene_Field::Text('body', $docBody, $this->_doc->actualEncoding)); + } else { + $this->addField(Zend_Search_Lucene_Field::UnStored('body', $docBody, $this->_doc->actualEncoding)); + } + + $linkNodes = $this->_doc->getElementsByTagName('a'); + foreach ($linkNodes as $linkNode) { + if (($href = $linkNode->getAttribute('href')) != '') { + $this->_links[] = $href; + } + } + $this->_links = array_unique($this->_links); + + $linkNodes = $xpath->query('/html/head/link'); + foreach ($linkNodes as $linkNode) { + if (($href = $linkNode->getAttribute('href')) != '') { + $this->_headerLinks[] = $href; + } + } + $this->_headerLinks = array_unique($this->_headerLinks); + } + + /** + * Get node text + * + * We should exclude scripts, which may be not included into comment tags, CDATA sections, + * + * @param DOMNode $node + * @param string &$text + */ + private function _retrieveNodeText(DOMNode $node, &$text) + { + if ($node->nodeType == XML_TEXT_NODE) { + $text .= $node->nodeValue ; + $text .= ' '; + } else if ($node->nodeType == XML_ELEMENT_NODE && $node->nodeName != 'script') { + foreach ($node->childNodes as $childNode) { + $this->_retrieveNodeText($childNode, $text); + } + } + } + + /** + * Get document HREF links + * + * @return array + */ + public function getLinks() + { + return $this->_links; + } + + /** + * Get document header links + * + * @return array + */ + public function getHeaderLinks() + { + return $this->_headerLinks; + } + + /** + * Load HTML document from a string + * + * @param string $data + * @param boolean $storeContent + * @return Zend_Search_Lucene_Document_Html + */ + public static function loadHTML($data, $storeContent = false) + { + return new Zend_Search_Lucene_Document_Html($data, false, $storeContent); + } + + /** + * Load HTML document from a file + * + * @param string $file + * @param boolean $storeContent + * @return Zend_Search_Lucene_Document_Html + */ + public static function loadHTMLFile($file, $storeContent = false) + { + return new Zend_Search_Lucene_Document_Html($file, true, $storeContent); + } + + + /** + * Highlight text in text node + * + * @param DOMText $node + * @param array $wordsToHighlight + * @param string $color + */ + public function _highlightTextNode(DOMText $node, $wordsToHighlight, $color) + { + $analyzer = Zend_Search_Lucene_Analysis_Analyzer::getDefault(); + $analyzer->setInput($node->nodeValue, $this->_doc->encoding); + + $matchedTokens = array(); + + while (($token = $analyzer->nextToken()) !== null) { + if (isset($wordsToHighlight[$token->getTermText()])) { + $matchedTokens[] = $token; + } + } + + if (count($matchedTokens) == 0) { + return; + } + + $matchedTokens = array_reverse($matchedTokens); + + foreach ($matchedTokens as $token) { + // Cut text after matched token + $node->splitText($token->getEndOffset()); + + // Cut matched node + $matchedWordNode = $node->splitText($token->getStartOffset()); + + $highlightedNode = $this->_doc->createElement('b', $matchedWordNode->nodeValue); + $highlightedNode->setAttribute('style', 'color:black;background-color:' . $color); + + $node->parentNode->replaceChild($highlightedNode, $matchedWordNode); + } + } + + + /** + * highlight words in content of the specified node + * + * @param DOMNode $contextNode + * @param array $wordsToHighlight + * @param string $color + */ + public function _highlightNode(DOMNode $contextNode, $wordsToHighlight, $color) + { + $textNodes = array(); + + if (!$contextNode->hasChildNodes()) { + return; + } + + foreach ($contextNode->childNodes as $childNode) { + if ($childNode->nodeType == XML_TEXT_NODE) { + // process node later to leave childNodes structure untouched + $textNodes[] = $childNode; + } else { + // Skip script nodes + if ($childNode->nodeName != 'script') { + $this->_highlightNode($childNode, $wordsToHighlight, $color); + } + } + } + + foreach ($textNodes as $textNode) { + $this->_highlightTextNode($textNode, $wordsToHighlight, $color); + } + } + + + + /** + * Highlight text with specified color + * + * @param string|array $words + * @param string $color + * @return string + */ + public function highlight($words, $color = '#66ffff') + { + if (!is_array($words)) { + $words = array($words); + } + $wordsToHighlight = array(); + + $analyzer = Zend_Search_Lucene_Analysis_Analyzer::getDefault(); + foreach ($words as $wordString) { + $wordsToHighlight = array_merge($wordsToHighlight, $analyzer->tokenize($wordString)); + } + + if (count($wordsToHighlight) == 0) { + return $this->_doc->saveHTML(); + } + + $wordsToHighlightFlipped = array(); + foreach ($wordsToHighlight as $id => $token) { + $wordsToHighlightFlipped[$token->getTermText()] = $id; + } + + $xpath = new DOMXPath($this->_doc); + + $matchedNodes = $xpath->query("/html/body/*"); + foreach ($matchedNodes as $matchedNode) { + $this->_highlightNode($matchedNode, $wordsToHighlightFlipped, $color); + } + + } + + /** + * Get HTML + * + * @return string + */ + public function getHTML() + { + return $this->_doc->saveHTML(); + } +} + diff --git a/search/Zend/Search/Lucene/Exception.php b/search/Zend/Search/Lucene/Exception.php index 5b73b29c5ea..9d06e89522a 100644 --- a/search/Zend/Search/Lucene/Exception.php +++ b/search/Zend/Search/Lucene/Exception.php @@ -14,7 +14,7 @@ * * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -22,13 +22,13 @@ /** * Framework base exception */ -require_once 'Zend/Search/Exception.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Exception.php'; /** * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Exception extends Zend_Search_Exception diff --git a/search/Zend/Search/Lucene/FSM.php b/search/Zend/Search/Lucene/FSM.php new file mode 100644 index 00000000000..31c9069fe4c --- /dev/null +++ b/search/Zend/Search/Lucene/FSM.php @@ -0,0 +1,433 @@ +dirroot.'/search/Zend/Search/Lucene/FSMAction.php'; + +/** Zend_Search_Exception */ +require_once $CFG->dirroot.'/search/Zend/Search/Exception.php'; + + +/** + * Abstract Finite State Machine + * + * Take a look on Wikipedia state machine description: http://en.wikipedia.org/wiki/Finite_state_machine + * + * Any type of Transducers (Moore machine or Mealy machine) also may be implemented by using this abstract FSM. + * process() methods invokes a specified actions which may construct FSM output. + * Actions may be also used to signal, that we have reached Accept State + * + * @category Zend + * @package Zend_Search_Lucene + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +abstract class Zend_Search_Lucene_FSM +{ + /** + * Machine States alphabet + * + * @var array + */ + private $_states = array(); + + /** + * Current state + * + * @var integer|string + */ + private $_currentState = null; + + /** + * Input alphabet + * + * @var array + */ + private $_inputAphabet = array(); + + /** + * State transition table + * + * [sourceState][input] => targetState + * + * @var array + */ + private $_rules = array(); + + /** + * List of entry actions + * Each action executes when entering the state + * + * [state] => action + * + * @var array + */ + private $_entryActions = array(); + + /** + * List of exit actions + * Each action executes when exiting the state + * + * [state] => action + * + * @var array + */ + private $_exitActions = array(); + + /** + * List of input actions + * Each action executes when entering the state + * + * [state][input] => action + * + * @var array + */ + private $_inputActions = array(); + + /** + * List of input actions + * Each action executes when entering the state + * + * [state1][state2] => action + * + * @var array + */ + private $_transitionActions = array(); + + /** + * Finite State machine constructor + * + * $states is an array of integers or strings with a list of possible machine states + * constructor treats fist list element as a sturt state (assignes it to $_current state). + * It may be reassigned by setState() call. + * States list may be empty and can be extended later by addState() or addStates() calls. + * + * $inputAphabet is the same as $states, but represents input alphabet + * it also may be extended later by addInputSymbols() or addInputSymbol() calls. + * + * $rules parameter describes FSM transitions and has a structure: + * array( array(sourseState, input, targetState[, inputAction]), + * array(sourseState, input, targetState[, inputAction]), + * array(sourseState, input, targetState[, inputAction]), + * ... + * ) + * Rules also can be added later by addRules() and addRule() calls. + * + * FSM actions are very flexible and may be defined by addEntryAction(), addExitAction(), + * addInputAction() and addTransitionAction() calls. + * + * @param array $states + * @param array $inputAphabet + * @param array $rules + */ + public function __construct($states = array(), $inputAphabet = array(), $rules = array()) + { + $this->addStates($states); + $this->addInputSymbols($inputAphabet); + $this->addRules($rules); + } + + /** + * Add states to the state machine + * + * @param array $states + */ + public function addStates($states) + { + foreach ($states as $state) { + $this->addState($state); + } + } + + /** + * Add state to the state machine + * + * @param integer|string $state + */ + public function addState($state) + { + $this->_states[$state] = $state; + + if ($this->_currentState === null) { + $this->_currentState = $state; + } + } + + /** + * Set FSM state. + * No any action is invoked + * + * @param integer|string $state + * @throws Zend_Search_Exception + */ + public function setState($state) + { + if (!isset($this->_states[$state])) { + throw new Zend_Search_Exception('State \'' . $state . '\' is not on of the possible FSM states.'); + } + + $this->_currentState = $state; + } + + /** + * Get FSM state. + * + * @return integer|string $state|null + */ + public function getState() + { + return $this->_currentState; + } + + /** + * Add symbols to the input alphabet + * + * @param array $inputAphabet + */ + public function addInputSymbols($inputAphabet) + { + foreach ($inputAphabet as $inputSymbol) { + $this->addInputSymbol($inputSymbol); + } + } + + /** + * Add symbol to the input alphabet + * + * @param integer|string $inputSymbol + */ + public function addInputSymbol($inputSymbol) + { + $this->_inputAphabet[$inputSymbol] = $inputSymbol; + } + + + /** + * Add transition rules + * + * array structure: + * array( array(sourseState, input, targetState[, inputAction]), + * array(sourseState, input, targetState[, inputAction]), + * array(sourseState, input, targetState[, inputAction]), + * ... + * ) + * + * @param array $rules + */ + public function addRules($rules) + { + foreach ($rules as $rule) { + $this->addrule($rule[0], $rule[1], $rule[2], isset($rule[3])?$rule[3]:null); + } + } + + /** + * Add symbol to the input alphabet + * + * @param integer|string $sourceState + * @param integer|string $input + * @param integer|string $targetState + * @param Zend_Search_Lucene_FSMAction|null $inputAction + * @throws Zend_Search_Exception + */ + public function addRule($sourceState, $input, $targetState, $inputAction = null) + { + if (!isset($this->_states[$sourceState])) { + throw new Zend_Search_Exception('Undefined source state (' . $sourceState . ').'); + } + if (!isset($this->_states[$targetState])) { + throw new Zend_Search_Exception('Undefined target state (' . $targetState . ').'); + } + if (!isset($this->_inputAphabet[$input])) { + throw new Zend_Search_Exception('Undefined input symbol (' . $input . ').'); + } + + if (!isset($this->_rules[$sourceState])) { + $this->_rules[$sourceState] = array(); + } + if (isset($this->_rules[$sourceState][$input])) { + throw new Zend_Search_Exception('Rule for {state,input} pair (' . $sourceState . ', '. $input . ') is already defined.'); + } + + $this->_rules[$sourceState][$input] = $targetState; + + + if ($inputAction !== null) { + $this->addInputAction($sourceState, $input, $inputAction); + } + } + + + /** + * Add state entry action. + * Several entry actions are allowed. + * Action execution order is defined by addEntryAction() calls + * + * @param integer|string $state + * @param Zend_Search_Lucene_FSMAction $action + */ + public function addEntryAction($state, Zend_Search_Lucene_FSMAction $action) + { + if (!isset($this->_states[$state])) { + throw new Zend_Search_Exception('Undefined state (' . $state. ').'); + } + + if (!isset($this->_entryActions[$state])) { + $this->_entryActions[$state] = array(); + } + + $this->_entryActions[$state][] = $action; + } + + /** + * Add state exit action. + * Several exit actions are allowed. + * Action execution order is defined by addEntryAction() calls + * + * @param integer|string $state + * @param Zend_Search_Lucene_FSMAction $action + */ + public function addExitAction($state, Zend_Search_Lucene_FSMAction $action) + { + if (!isset($this->_states[$state])) { + throw new Zend_Search_Exception('Undefined state (' . $state. ').'); + } + + if (!isset($this->_exitActions[$state])) { + $this->_exitActions[$state] = array(); + } + + $this->_exitActions[$state][] = $action; + } + + /** + * Add input action (defined by {state, input} pair). + * Several input actions are allowed. + * Action execution order is defined by addInputAction() calls + * + * @param integer|string $state + * @param integer|string $input + * @param Zend_Search_Lucene_FSMAction $action + */ + public function addInputAction($state, $inputSymbol, Zend_Search_Lucene_FSMAction $action) + { + if (!isset($this->_states[$state])) { + throw new Zend_Search_Exception('Undefined state (' . $state. ').'); + } + if (!isset($this->_inputAphabet[$inputSymbol])) { + throw new Zend_Search_Exception('Undefined input symbol (' . $inputSymbol. ').'); + } + + if (!isset($this->_inputActions[$state])) { + $this->_inputActions[$state] = array(); + } + if (!isset($this->_inputActions[$state][$inputSymbol])) { + $this->_inputActions[$state][$inputSymbol] = array(); + } + + $this->_inputActions[$state][$inputSymbol][] = $action; + } + + /** + * Add transition action (defined by {state, input} pair). + * Several transition actions are allowed. + * Action execution order is defined by addTransitionAction() calls + * + * @param integer|string $sourceState + * @param integer|string $targetState + * @param Zend_Search_Lucene_FSMAction $action + */ + public function addTransitionAction($sourceState, $targetState, Zend_Search_Lucene_FSMAction $action) + { + if (!isset($this->_states[$sourceState])) { + throw new Zend_Search_Exception('Undefined source state (' . $sourceState. ').'); + } + if (!isset($this->_states[$targetState])) { + throw new Zend_Search_Exception('Undefined source state (' . $targetState. ').'); + } + + if (!isset($this->_transitionActions[$sourceState])) { + $this->_transitionActions[$sourceState] = array(); + } + if (!isset($this->_transitionActions[$sourceState][$targetState])) { + $this->_transitionActions[$sourceState][$targetState] = array(); + } + + $this->_transitionActions[$sourceState][$targetState][] = $action; + } + + + /** + * Process an input + * + * @param mixed $input + * @throws Zend_Search_Exception + */ + public function process($input) + { + if (!isset($this->_rules[$this->_currentState])) { + throw new Zend_Search_Exception('There is no any rule for current state (' . $this->_currentState . ').'); + } + if (!isset($this->_rules[$this->_currentState][$input])) { + throw new Zend_Search_Exception('There is no any rule for {current state, input} pair (' . $this->_currentState . ', ' . $input . ').'); + } + + $sourceState = $this->_currentState; + $targetState = $this->_rules[$this->_currentState][$input]; + + if ($sourceState != $targetState && isset($this->_exitActions[$sourceState])) { + foreach ($this->_exitActions[$sourceState] as $action) { + $action->doAction(); + } + } + if (isset($this->_inputActions[$sourceState]) && + isset($this->_inputActions[$sourceState][$input])) { + foreach ($this->_inputActions[$sourceState][$input] as $action) { + $action->doAction(); + } + } + + + $this->_currentState = $targetState; + + if (isset($this->_transitionActions[$sourceState]) && + isset($this->_transitionActions[$sourceState][$targetState])) { + foreach ($this->_transitionActions[$sourceState][$targetState] as $action) { + $action->doAction(); + } + } + if ($sourceState != $targetState && isset($this->_entryActions[$targetState])) { + foreach ($this->_entryActions[$targetState] as $action) { + $action->doAction(); + } + } + } + + public function reset() + { + if (count($this->_states) == 0) { + throw new Zend_Search_Exception('There is no any state defined for FSM.'); + } + + $this->_currentState = $this->_states[0]; + } +} + diff --git a/search/Zend/Search/Lucene/FSMAction.php b/search/Zend/Search/Lucene/FSMAction.php new file mode 100644 index 00000000000..606de7b613f --- /dev/null +++ b/search/Zend/Search/Lucene/FSMAction.php @@ -0,0 +1,65 @@ +_object = $object; + $this->_method = $method; + } + + public function doAction() + { + $methodName = $this->_method; + $this->_object->$methodName(); + } +} + diff --git a/search/Zend/Search/Lucene/Field.php b/search/Zend/Search/Lucene/Field.php index 5a18fcfc468..86cd22ccea1 100644 --- a/search/Zend/Search/Lucene/Field.php +++ b/search/Zend/Search/Lucene/Field.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Document - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -31,15 +31,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Document - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Field { - public $kind; + /** + * Field name + * + * @var string + */ + public $name; - public $name = 'body'; - public $stringValue = null; + + public $value; public $isStored = false; public $isIndexed = true; public $isTokenized = true; @@ -47,26 +52,48 @@ class Zend_Search_Lucene_Field public $storeTermVector = false; + /** + * Field boos factor + * It's not stored directly in the index, but affects on normalizetion factor + * + * @var float + */ public $boost = 1.0; - public function __construct($name, $stringValue, $isStored, $isIndexed, $isTokenized, $isBinary = false) + /** + * Field value encoding. + * + * @var string + */ + public $encoding; + + /** + * Object constructor + * + * @param string $name + * @param string $value + * @param string $encoding + * @param boolean $isStored + * @param boolean $isIndexed + * @param boolean $isTokenized + * @param boolean $isBinary + */ + public function __construct($name, $value, $encoding, $isStored, $isIndexed, $isTokenized, $isBinary = false) { - $this->name = $name; + $this->name = $name; + $this->value = $value; if (!$isBinary) { - /** - * @todo Correct UTF-8 string should be required in future - * Until full UTF-8 support is not completed, string should be normalized to ANSII encoding - */ - $this->stringValue = iconv(mb_detect_encoding($stringValue), 'ASCII//TRANSLIT', $stringValue); - //$this->stringValue = iconv('', 'ASCII//TRANSLIT', $stringValue); + $this->encoding = $encoding; + $this->isTokenized = $isTokenized; } else { - $this->stringValue = $stringValue; + $this->encoding = ''; + $this->isTokenized = false; } - $this->isStored = $isStored; - $this->isIndexed = $isIndexed; - $this->isTokenized = $isTokenized; - $this->isBinary = $isBinary; + + $this->isStored = $isStored; + $this->isIndexed = $isIndexed; + $this->isBinary = $isBinary; $this->storeTermVector = false; $this->boost = 1.0; @@ -79,11 +106,12 @@ class Zend_Search_Lucene_Field * * @param string $name * @param string $value + * @param string $encoding * @return Zend_Search_Lucene_Field */ - static public function Keyword($name, $value) + public static function Keyword($name, $value, $encoding = '') { - return new self($name, $value, true, true, false); + return new self($name, $value, $encoding, true, true, false); } @@ -93,11 +121,12 @@ class Zend_Search_Lucene_Field * * @param string $name * @param string $value + * @param string $encoding * @return Zend_Search_Lucene_Field */ - static public function UnIndexed($name, $value) + public static function UnIndexed($name, $value, $encoding = '') { - return new self($name, $value, true, false, false); + return new self($name, $value, $encoding, true, false, false); } @@ -107,11 +136,12 @@ class Zend_Search_Lucene_Field * * @param string $name * @param string $value + * @param string $encoding * @return Zend_Search_Lucene_Field */ - static public function Binary($name, $value) + public static function Binary($name, $value) { - return new self($name, $value, true, false, false, true); + return new self($name, $value, '', true, false, false, true); } /** @@ -121,11 +151,12 @@ class Zend_Search_Lucene_Field * * @param string $name * @param string $value + * @param string $encoding * @return Zend_Search_Lucene_Field */ - static public function Text($name, $value) + public static function Text($name, $value, $encoding = '') { - return new self($name, $value, true, true, true); + return new self($name, $value, $encoding, true, true, true); } @@ -135,12 +166,27 @@ class Zend_Search_Lucene_Field * * @param string $name * @param string $value + * @param string $encoding * @return Zend_Search_Lucene_Field */ - static public function UnStored($name, $value) + public static function UnStored($name, $value, $encoding = '') { - return new self($name, $value, false, true, true); + return new self($name, $value, $encoding, false, true, true); } + /** + * Get field value in UTF-8 encoding + * + * @return string + */ + public function getUtf8Value() + { + if (strcasecmp($this->encoding, 'utf8' ) == 0 || + strcasecmp($this->encoding, 'utf-8') == 0 ) { + return $this->value; + } else { + return iconv($this->encoding, 'UTF-8', $this->value); + } + } } diff --git a/search/Zend/Search/Lucene/Index/DictionaryLoader.php b/search/Zend/Search/Lucene/Index/DictionaryLoader.php new file mode 100644 index 00000000000..d3f0669c814 --- /dev/null +++ b/search/Zend/Search/Lucene/Index/DictionaryLoader.php @@ -0,0 +1,254 @@ +dirroot.'/search/Zend/Search/Lucene/Exception.php'; + + +/** + * Dictionary loader + * + * It's a dummy class which is created to encapsulate non-good structured code. + * Manual "method inlining" is performed to increase dictionary index loading operation + * which is major bottelneck for search performance. + * + * + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Index + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Index_DictionaryLoader +{ + /** + * Dictionary index loader. + * + * It takes a string which is actually .tii index file data and + * returns two arrays - term and tremInfo lists. + * + * See Zend_Search_Lucene_Index_SegmintInfo class for details + * + * @param string $data + * @return array + * @throws Zend_Search_Lucene_Exception + */ + public static function load($data) + { + $termDictionary = array(); + $termInfos = array(); + $pos = 0; + + // $tiVersion = $tiiFile->readInt(); + $tiVersion = ord($data[0]) << 24 | ord($data[1]) << 16 | ord($data[2]) << 8 | ord($data[3]); + $pos += 4; + if ($tiVersion != (int)0xFFFFFFFE) { + throw new Zend_Search_Lucene_Exception('Wrong TermInfoIndexFile file format'); + } + + // $indexTermCount = = $tiiFile->readLong(); + if (PHP_INT_SIZE > 4) { + $indexTermCount = ord($data[$pos]) << 56 | + ord($data[$pos+1]) << 48 | + ord($data[$pos+2]) << 40 | + ord($data[$pos+3]) << 32 | + ord($data[$pos+4]) << 24 | + ord($data[$pos+5]) << 16 | + ord($data[$pos+6]) << 8 | + ord($data[$pos+7]); + } else { + if ((ord($data[$pos]) != 0) || + (ord($data[$pos+1]) != 0) || + (ord($data[$pos+2]) != 0) || + (ord($data[$pos+3]) != 0) || + ((ord($data[$pos+4]) & 0x80) != 0)) { + throw new Zend_Search_Lucene_Exception('Largest supported segment size (for 32-bit mode) is 2Gb'); + } + + $indexTermCount = ord($data[$pos+4]) << 24 | + ord($data[$pos+5]) << 16 | + ord($data[$pos+6]) << 8 | + ord($data[$pos+7]); + } + $pos += 8; + + // $tiiFile->readInt(); // IndexInterval + $pos += 4; + + // $skipInterval = $tiiFile->readInt(); + $skipInterval = ord($data[$pos]) << 24 | ord($data[$pos+1]) << 16 | ord($data[$pos+2]) << 8 | ord($data[$pos+3]); + $pos += 4; + if ($indexTermCount < 1) { + throw new Zend_Search_Lucene_Exception('Wrong number of terms in a term dictionary index'); + } + + $prevTerm = ''; + $freqPointer = 0; + $proxPointer = 0; + $indexPointer = 0; + for ($count = 0; $count < $indexTermCount; $count++) { + //$termPrefixLength = $tiiFile->readVInt(); + $nbyte = ord($data[$pos++]); + $termPrefixLength = $nbyte & 0x7F; + for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) { + $nbyte = ord($data[$pos++]); + $termPrefixLength |= ($nbyte & 0x7F) << $shift; + } + + // $termSuffix = $tiiFile->readString(); + $nbyte = ord($data[$pos++]); + $len = $nbyte & 0x7F; + for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) { + $nbyte = ord($data[$pos++]); + $len |= ($nbyte & 0x7F) << $shift; + } + if ($len == 0) { + $termSuffix = ''; + } else { + $termSuffix = substr($data, $pos, $len); + $pos += $len; + for ($count1 = 0; $count1 < $len; $count1++ ) { + if (( ord($termSuffix[$count1]) & 0xC0 ) == 0xC0) { + $addBytes = 1; + if (ord($termSuffix[$count1]) & 0x20 ) { + $addBytes++; + } + $termSuffix .= substr($data, $pos, $addBytes); + $pos += $addBytes; + $len += $addBytes; + + // Check for null character. Java2 encodes null character + // in two bytes. + if (ord($termSuffix[$count1]) == 0xC0 && + ord($termSuffix[$count1+1]) == 0x80 ) { + $termSuffix[$count1] = 0; + $termSuffix = substr($termSuffix,0,$count1+1) + . substr($termSuffix,$count1+2); + } + $count1 += $addBytes; + } + } + } + + // $termValue = Zend_Search_Lucene_Index_Term::getPrefix($prevTerm, $termPrefixLength) . $termSuffix; + $pb = 0; $pc = 0; + while ($pb < strlen($prevTerm) && $pc < $termPrefixLength) { + $charBytes = 1; + if ((ord($prevTerm[$pb]) & 0xC0) == 0xC0) { + $charBytes++; + if (ord($prevTerm[$pb]) & 0x20 ) { + $charBytes++; + if (ord($prevTerm[$pb]) & 0x10 ) { + $charBytes++; + } + } + } + + if ($pb + $charBytes > strlen($data)) { + // wrong character + break; + } + + $pc++; + $pb += $charBytes; + } + $termValue = substr($prevTerm, 0, $pb) . $termSuffix; + + // $termFieldNum = $tiiFile->readVInt(); + $nbyte = ord($data[$pos++]); + $termFieldNum = $nbyte & 0x7F; + for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) { + $nbyte = ord($data[$pos++]); + $termFieldNum |= ($nbyte & 0x7F) << $shift; + } + + // $docFreq = $tiiFile->readVInt(); + $nbyte = ord($data[$pos++]); + $docFreq = $nbyte & 0x7F; + for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) { + $nbyte = ord($data[$pos++]); + $docFreq |= ($nbyte & 0x7F) << $shift; + } + + // $freqPointer += $tiiFile->readVInt(); + $nbyte = ord($data[$pos++]); + $vint = $nbyte & 0x7F; + for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) { + $nbyte = ord($data[$pos++]); + $vint |= ($nbyte & 0x7F) << $shift; + } + $freqPointer += $vint; + + // $proxPointer += $tiiFile->readVInt(); + $nbyte = ord($data[$pos++]); + $vint = $nbyte & 0x7F; + for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) { + $nbyte = ord($data[$pos++]); + $vint |= ($nbyte & 0x7F) << $shift; + } + $proxPointer += $vint; + + if( $docFreq >= $skipInterval ) { + // $skipDelta = $tiiFile->readVInt(); + $nbyte = ord($data[$pos++]); + $vint = $nbyte & 0x7F; + for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) { + $nbyte = ord($data[$pos++]); + $vint |= ($nbyte & 0x7F) << $shift; + } + $skipDelta = $vint; + } else { + $skipDelta = 0; + } + + // $indexPointer += $tiiFile->readVInt(); + $nbyte = ord($data[$pos++]); + $vint = $nbyte & 0x7F; + for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) { + $nbyte = ord($data[$pos++]); + $vint |= ($nbyte & 0x7F) << $shift; + } + $indexPointer += $vint; + + + // $this->_termDictionary[] = new Zend_Search_Lucene_Index_Term($termValue, $termFieldNum); + $termDictionary[] = array($termFieldNum, $termValue); + + $termInfos[] = + // new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipDelta, $indexPointer); + array($docFreq, $freqPointer, $proxPointer, $skipDelta, $indexPointer); + + $prevTerm = $termValue; + } + + // Check special index entry mark + if ($termDictionary[0][0] != (int)0xFFFFFFFF) { + throw new Zend_Search_Lucene_Exception('Wrong TermInfoIndexFile file format'); + } else if (PHP_INT_SIZE > 4){ + // Treat 64-bit 0xFFFFFFFF as -1 + $termDictionary[0][0] = -1; + } + + return array(&$termDictionary, &$termInfos); + } +} + diff --git a/search/Zend/Search/Lucene/Index/FieldInfo.php b/search/Zend/Search/Lucene/Index/FieldInfo.php index 4c11aaac77c..1d138b6575f 100644 --- a/search/Zend/Search/Lucene/Index/FieldInfo.php +++ b/search/Zend/Search/Lucene/Index/FieldInfo.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -24,7 +24,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_FieldInfo diff --git a/search/Zend/Search/Lucene/Index/SegmentInfo.php b/search/Zend/Search/Lucene/Index/SegmentInfo.php index aeceab63b62..c6f7868fd50 100644 --- a/search/Zend/Search/Lucene/Index/SegmentInfo.php +++ b/search/Zend/Search/Lucene/Index/SegmentInfo.php @@ -15,20 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ +/** Zend_Search_Lucene_Index_DictionaryLoader */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/DictionaryLoader.php'; + /** Zend_Search_Lucene_Exception */ -require_once 'Zend/Search/Lucene/Exception.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_SegmentInfo @@ -49,7 +52,12 @@ class Zend_Search_Lucene_Index_SegmentInfo /** * Term Dictionary Index - * Array of the Zend_Search_Lucene_Index_Term objects + * + * Array of arrays (Zend_Search_Lucene_Index_Term objects are represented as arrays because + * of performance considerations) + * [0] -> $termValue + * [1] -> $termFieldNum + * * Corresponding Zend_Search_Lucene_Index_TermInfo object stored in the $_termDictionaryInfos * * @var array @@ -58,7 +66,14 @@ class Zend_Search_Lucene_Index_SegmentInfo /** * Term Dictionary Index TermInfos - * Array of the Zend_Search_Lucene_Index_TermInfo objects + * + * Array of arrays (Zend_Search_Lucene_Index_TermInfo objects are represented as arrays because + * of performance considerations) + * [0] -> $docFreq + * [1] -> $freqPointer + * [2] -> $proxPointer + * [3] -> $skipOffset + * [4] -> $indexPointer * * @var array */ @@ -88,6 +103,14 @@ class Zend_Search_Lucene_Index_SegmentInfo */ private $_segFiles; + /** + * Associative array where the key is the file name and the value is file size (.csf). + * + * @var array + */ + private $_segFileSizes; + + /** * File system adapter. * @@ -122,6 +145,7 @@ class Zend_Search_Lucene_Index_SegmentInfo */ private $_deletedDirty = false; + /** * Zend_Search_Lucene_Index_SegmentInfo constructor needs Segmentname, * Documents count and Directory as a parameter. @@ -144,9 +168,15 @@ class Zend_Search_Lucene_Index_SegmentInfo for ($count = 0; $count < $segFilesCount; $count++) { $dataOffset = $cfsFile->readLong(); + if ($count != 0) { + $this->_segFileSizes[$fileName] = $dataOffset - end($this->_segFiles); + } $fileName = $cfsFile->readString(); $this->_segFiles[$fileName] = $dataOffset; } + if ($count != 0) { + $this->_segFileSizes[$fileName] = $this->_directory->fileLength($name . '.cfs') - $dataOffset; + } } $fnmFile = $this->openCompoundFile('.fnm'); @@ -197,7 +227,6 @@ class Zend_Search_Lucene_Index_SegmentInfo } } } - } } catch(Zend_Search_Exception $e) { if (strpos($e->getMessage(), 'compound file doesn\'t contain') !== false ) { @@ -212,16 +241,17 @@ class Zend_Search_Lucene_Index_SegmentInfo * Opens index file stoted within compound index file * * @param string $extension + * @param boolean $shareHandler * @throws Zend_Search_Lucene_Exception * @return Zend_Search_Lucene_Storage_File */ - public function openCompoundFile($extension) + public function openCompoundFile($extension, $shareHandler = true) { $filename = $this->_name . $extension; // Try to open common file first if ($this->_directory->fileExists($filename)) { - return $this->_directory->getFileObject($filename); + return $this->_directory->getFileObject($filename, $shareHandler); } if( !isset($this->_segFiles[$filename]) ) { @@ -229,11 +259,34 @@ class Zend_Search_Lucene_Index_SegmentInfo . $filename . ' file.' ); } - $file = $this->_directory->getFileObject( $this->_name.".cfs" ); + $file = $this->_directory->getFileObject($this->_name . '.cfs', $shareHandler); $file->seek($this->_segFiles[$filename]); return $file; } + /** + * Get compound file length + * + * @param string $extension + * @return integer + */ + public function compoundFileLength($extension) + { + $filename = $this->_name . $extension; + + // Try to get common file first + if ($this->_directory->fileExists($filename)) { + return $this->_directory->fileLength($filename); + } + + if( !isset($this->_segFileSizes[$filename]) ) { + throw new Zend_Search_Lucene_Exception('Index compound file doesn\'t contain ' + . $filename . ' file.' ); + } + + return $this->_segFileSizes[$filename]; + } + /** * Returns field index or -1 if field is not found * @@ -255,7 +308,7 @@ class Zend_Search_Lucene_Index_SegmentInfo * Returns field info for specified field * * @param integer $fieldNum - * @return ZSearchFieldInfo + * @return Zend_Search_Lucene_Index_FieldInfo */ public function getField($fieldNum) { @@ -281,7 +334,17 @@ class Zend_Search_Lucene_Index_SegmentInfo } /** - * Returns the total number of documents in this segment. + * Returns array of FieldInfo objects. + * + * @return array + */ + public function getFieldInfos() + { + return $this->_fields; + } + + /** + * Returns the total number of documents in this segment (including deleted documents). * * @return integer */ @@ -290,6 +353,38 @@ class Zend_Search_Lucene_Index_SegmentInfo return $this->_docCount; } + /** + * Returns number of deleted documents. + * + * @return integer + */ + private function _deletedCount() + { + if ($this->_deleted === null) { + return 0; + } + + if (extension_loaded('bitset')) { + return count(bitset_to_array($this->_deleted)); + } else { + return count($this->_deleted); + } + } + + /** + * Returns the total number of non-deleted documents in this segment. + * + * @return integer + */ + public function numDocs() + { + if ($this->hasDeletions()) { + return $this->_docCount - $this->_deletedCount(); + } else { + return $this->_docCount; + } + } + /** * Get field position in a fields dictionary * @@ -302,57 +397,6 @@ class Zend_Search_Lucene_Index_SegmentInfo $this->_fieldsDicPositions[$fieldNum] : $fieldNum; } - /** - * Loads Term dictionary from TermInfoIndex file - */ - protected function _loadDictionary() - { - if ($this->_termDictionary !== null) { - return; - } - - $this->_termDictionary = array(); - $this->_termDictionaryInfos = array(); - - $tiiFile = $this->openCompoundFile('.tii'); - $tiVersion = $tiiFile->readInt(); - if ($tiVersion != (int)0xFFFFFFFE) { - throw new Zend_Search_Lucene_Exception('Wrong TermInfoIndexFile file format'); - } - - $indexTermCount = $tiiFile->readLong(); - $tiiFile->readInt(); // IndexInterval - $skipInterval = $tiiFile->readInt(); - - $prevTerm = ''; - $freqPointer = 0; - $proxPointer = 0; - $indexPointer = 0; - for ($count = 0; $count < $indexTermCount; $count++) { - $termPrefixLength = $tiiFile->readVInt(); - $termSuffix = $tiiFile->readString(); - $termValue = substr( $prevTerm, 0, $termPrefixLength ) . $termSuffix; - - $termFieldNum = $tiiFile->readVInt(); - $docFreq = $tiiFile->readVInt(); - $freqPointer += $tiiFile->readVInt(); - $proxPointer += $tiiFile->readVInt(); - if( $docFreq >= $skipInterval ) { - $skipDelta = $tiiFile->readVInt(); - } else { - $skipDelta = 0; - } - - $indexPointer += $tiiFile->readVInt(); - - $this->_termDictionary[] = new Zend_Search_Lucene_Index_Term($termValue,$termFieldNum); - $this->_termDictionaryInfos[] = - new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipDelta, $indexPointer); - $prevTerm = $termValue; - } - } - - /** * Return segment name * @@ -364,15 +408,75 @@ class Zend_Search_Lucene_Index_SegmentInfo } + /** + * TermInfo cache + * + * Size is 1024. + * Numbers are used instead of class constants because of performance considerations + * + * @var array + */ + private $_termInfoCache = array(); + + private function _cleanUpTermInfoCache() + { + // Clean 256 term infos + foreach ($this->_termInfoCache as $key => $termInfo) { + unset($this->_termInfoCache[$key]); + + // leave 768 last used term infos + if (count($this->_termInfoCache) == 768) { + break; + } + } + } + /** * Scans terms dictionary and returns term info * * @param Zend_Search_Lucene_Index_Term $term * @return Zend_Search_Lucene_Index_TermInfo */ - public function getTermInfo($term) + public function getTermInfo(Zend_Search_Lucene_Index_Term $term) { - $this->_loadDictionary(); + $termKey = $term->key(); + if (isset($this->_termInfoCache[$termKey])) { + $termInfo = $this->_termInfoCache[$termKey]; + + // Move termInfo to the end of cache + unset($this->_termInfoCache[$termKey]); + $this->_termInfoCache[$termKey] = $termInfo; + + return $termInfo; + } + + + if ($this->_termDictionary === null) { + // Check, if index is already serialized + if ($this->_directory->fileExists($this->_name . '.sti')) { + // Prefetch dictionary index data + $stiFile = $this->_directory->getFileObject($this->_name . '.sti'); + $stiFileData = $stiFile->readBytes($this->_directory->fileLength($this->_name . '.sti')); + + // Load dictionary index data + list($this->_termDictionary, $this->_termDictionaryInfos) = unserialize($stiFileData); + } else { + // Prefetch dictionary index data + $tiiFile = $this->openCompoundFile('.tii'); + $tiiFileData = $tiiFile->readBytes($this->compoundFileLength('.tii')); + + // Load dictionary index data + list($this->_termDictionary, $this->_termDictionaryInfos) = + Zend_Search_Lucene_Index_DictionaryLoader::load($tiiFileData); + + $stiFileData = serialize(array($this->_termDictionary, $this->_termDictionaryInfos)); + $stiFile = $this->_directory->createFile($this->_name . '.sti'); + $stiFile->writeBytes($stiFileData); + } + + } + + $searchField = $this->getFieldNum($term->field); @@ -389,10 +493,10 @@ class Zend_Search_Lucene_Index_SegmentInfo $mid = ($highIndex + $lowIndex) >> 1; $midTerm = $this->_termDictionary[$mid]; - $fieldNum = $this->_getFieldPosition($midTerm->field); + $fieldNum = $this->_getFieldPosition($midTerm[0] /* field */); $delta = $searchDicField - $fieldNum; if ($delta == 0) { - $delta = strcmp($term->text, $midTerm->text); + $delta = strcmp($term->text, $midTerm[1] /* text */); } if ($delta < 0) { @@ -400,7 +504,14 @@ class Zend_Search_Lucene_Index_SegmentInfo } elseif ($delta > 0) { $lowIndex = $mid+1; } else { - return $this->_termDictionaryInfos[$mid]; // We got it! + // return $this->_termDictionaryInfos[$mid]; // We got it! + $a = $this->_termDictionaryInfos[$mid]; + $termInfo = new Zend_Search_Lucene_Index_TermInfo($a[0], $a[1], $a[2], $a[3], $a[4]); + + // Put loaded termInfo into cache + $this->_termInfoCache[$termKey] = $termInfo; + + return $termInfo; } } @@ -411,7 +522,7 @@ class Zend_Search_Lucene_Index_SegmentInfo $prevPosition = $highIndex; $prevTerm = $this->_termDictionary[$prevPosition]; - $prevTermInfo = $this->_termDictionaryInfos[ $prevPosition ]; + $prevTermInfo = $this->_termDictionaryInfos[$prevPosition]; $tisFile = $this->openCompoundFile('.tis'); $tiVersion = $tisFile->readInt(); @@ -423,12 +534,12 @@ class Zend_Search_Lucene_Index_SegmentInfo $indexInterval = $tisFile->readInt(); $skipInterval = $tisFile->readInt(); - $tisFile->seek($prevTermInfo->indexPointer - 20 /* header size*/, SEEK_CUR); + $tisFile->seek($prevTermInfo[4] /* indexPointer */ - 20 /* header size*/, SEEK_CUR); - $termValue = $prevTerm->text; - $termFieldNum = $prevTerm->field; - $freqPointer = $prevTermInfo->freqPointer; - $proxPointer = $prevTermInfo->proxPointer; + $termValue = $prevTerm[1] /* text */; + $termFieldNum = $prevTerm[0] /* field */; + $freqPointer = $prevTermInfo[1] /* freqPointer */; + $proxPointer = $prevTermInfo[2] /* proxPointer */; for ($count = $prevPosition*$indexInterval + 1; $count <= $termCount && ( $this->_getFieldPosition($termFieldNum) < $searchDicField || @@ -438,7 +549,7 @@ class Zend_Search_Lucene_Index_SegmentInfo $termPrefixLength = $tisFile->readVInt(); $termSuffix = $tisFile->readString(); $termFieldNum = $tisFile->readVInt(); - $termValue = substr( $termValue, 0, $termPrefixLength ) . $termSuffix; + $termValue = Zend_Search_Lucene_Index_Term::getPrefix($termValue, $termPrefixLength) . $termSuffix; $docFreq = $tisFile->readVInt(); $freqPointer += $tisFile->readVInt(); @@ -451,10 +562,115 @@ class Zend_Search_Lucene_Index_SegmentInfo } if ($termFieldNum == $searchField && $termValue == $term->text) { - return new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipOffset); + $termInfo = new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipOffset); } else { - return null; + $termInfo = null; } + + // Put loaded termInfo into cache + $this->_termInfoCache[$termKey] = $termInfo; + + if (count($this->_termInfoCache) == 1024) { + $this->_cleanUpTermInfoCache(); + } + + return $termInfo; + } + + /** + * Returns term freqs array. + * Result array structure: array(docId => freq, ...) + * + * @param Zend_Search_Lucene_Index_Term $term + * @param integer $shift + * @return Zend_Search_Lucene_Index_TermInfo + */ + public function termFreqs(Zend_Search_Lucene_Index_Term $term, $shift = 0) + { + $termInfo = $this->getTermInfo($term); + + if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) { + return array(); + } + + $frqFile = $this->openCompoundFile('.frq'); + $frqFile->seek($termInfo->freqPointer,SEEK_CUR); + $result = array(); + $docId = 0; + + for ($count = 0; $count < $termInfo->docFreq; $count++) { + $docDelta = $frqFile->readVInt(); + if ($docDelta % 2 == 1) { + $docId += ($docDelta-1)/2; + $result[$shift + $docId] = 1; + } else { + $docId += $docDelta/2; + $result[$shift + $docId] = $frqFile->readVInt(); + } + } + + return $result; + } + + /** + * Returns term positions array. + * Result array structure: array(docId => array(pos1, pos2, ...), ...) + * + * @param Zend_Search_Lucene_Index_Term $term + * @param integer $shift + * @return Zend_Search_Lucene_Index_TermInfo + */ + public function termPositions(Zend_Search_Lucene_Index_Term $term, $shift = 0) + { + $termInfo = $this->getTermInfo($term); + + if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) { + return array(); + } + + $frqFile = $this->openCompoundFile('.frq'); + $frqFile->seek($termInfo->freqPointer,SEEK_CUR); + $freqs = array(); + $docId = 0; + + for ($count = 0; $count < $termInfo->docFreq; $count++) { + $docDelta = $frqFile->readVInt(); + if ($docDelta % 2 == 1) { + $docId += ($docDelta-1)/2; + $freqs[$docId] = 1; + } else { + $docId += $docDelta/2; + $freqs[$docId] = $frqFile->readVInt(); + } + } + + $result = array(); + $prxFile = $this->openCompoundFile('.prx'); + $prxFile->seek($termInfo->proxPointer, SEEK_CUR); + foreach ($freqs as $docId => $freq) { + $termPosition = 0; + $positions = array(); + + for ($count = 0; $count < $freq; $count++ ) { + $termPosition += $prxFile->readVInt(); + $positions[] = $termPosition; + } + + $result[$shift + $docId] = $positions; + } + + return $result; + } + + /** + * Load normalizatin factors from an index file + * + * @param integer $fieldNum + */ + private function _loadNorm($fieldNum) + { + $fFile = $this->openCompoundFile('.f' . $fieldNum); + $this->_norms[$fieldNum] = $fFile->readBytes($this->_docCount); } /** @@ -462,7 +678,7 @@ class Zend_Search_Lucene_Index_SegmentInfo * * @param integer $id * @param string $fieldName - * @return string + * @return float */ public function norm($id, $fieldName) { @@ -472,14 +688,37 @@ class Zend_Search_Lucene_Index_SegmentInfo return null; } - if ( !isset( $this->_norms[$fieldNum] )) { - $fFile = $this->openCompoundFile('.f' . $fieldNum); - $this->_norms[$fieldNum] = $fFile->readBytes($this->_docCount); + if (!isset($this->_norms[$fieldNum])) { + $this->_loadNorm($fieldNum); } return Zend_Search_Lucene_Search_Similarity::decodeNorm( ord($this->_norms[$fieldNum]{$id}) ); } + /** + * Returns norm vector, encoded in a byte string + * + * @param string $fieldName + * @return string + */ + public function normVector($fieldName) + { + $fieldNum = $this->getFieldNum($fieldName); + + if ($fieldNum == -1 || !($this->_fields[$fieldNum]->isIndexed)) { + $similarity = Zend_Search_Lucene_Search_Similarity::getDefault(); + + return str_repeat(chr($similarity->encodeNorm( $similarity->lengthNorm($fieldName, 0) )), + $this->_docCount); + } + + if (!isset($this->_norms[$fieldNum])) { + $this->_loadNorm($fieldNum); + } + + return $this->_norms[$fieldNum]; + } + /** * Returns true if any documents have been deleted from this index segment. @@ -571,5 +810,247 @@ class Zend_Search_Lucene_Index_SegmentInfo $this->_deletedDirty = false; } + + + + /** + * Term Dictionary File object for stream like terms reading + * + * @var Zend_Search_Lucene_Storage_File + */ + private $_tisFile = null; + + /** + * Frequencies File object for stream like terms reading + * + * @var Zend_Search_Lucene_Storage_File + */ + private $_frqFile = null; + + /** + * Offset of the .frq file in the compound file + * + * @var integer + */ + private $_frqFileOffset; + + /** + * Positions File object for stream like terms reading + * + * @var Zend_Search_Lucene_Storage_File + */ + private $_prxFile = null; + + /** + * Offset of the .prx file in the compound file + * + * @var integer + */ + private $_prxFileOffset; + + + /** + * Number of terms in term stream + * + * @var integer + */ + private $_termCount = 0; + + /** + * Segment skip interval + * + * @var integer + */ + private $_skipInterval; + + /** + * Last TermInfo in a terms stream + * + * @var Zend_Search_Lucene_Index_TermInfo + */ + private $_lastTermInfo = null; + + /** + * Last Term in a terms stream + * + * @var Zend_Search_Lucene_Index_Term + */ + private $_lastTerm = null; + + /** + * Map of the document IDs + * Used to get new docID after removing deleted documents. + * It's not very effective from memory usage point of view, + * but much more faster, then other methods + * + * @var array|null + */ + private $_docMap = null; + + /** + * An array of all term positions in the documents. + * Array structure: array( docId => array( pos1, pos2, ...), ...) + * + * @var array + */ + private $_lastTermPositions; + + /** + * Reset terms stream + * + * $startId - id for the fist document + * $compact - remove deleted documents + * + * Returns start document id for the next segment + * + * @param integer $startId + * @param boolean $compact + * @throws Zend_Search_Lucene_Exception + * @return integer + */ + public function reset($startId = 0, $compact = false) + { + if ($this->_tisFile !== null) { + $this->_tisFile = null; + } + + $this->_tisFile = $this->openCompoundFile('.tis', false); + $tiVersion = $this->_tisFile->readInt(); + if ($tiVersion != (int)0xFFFFFFFE) { + throw new Zend_Search_Lucene_Exception('Wrong TermInfoFile file format'); + } + + $this->_termCount = $this->_tisFile->readLong(); + $this->_tisFile->readInt(); // Read Index interval + $this->_skipInterval = $this->_tisFile->readInt(); // Read skip interval + + if ($this->_frqFile !== null) { + $this->_frqFile = null; + } + $this->_frqFile = $this->openCompoundFile('.frq', false); + $this->_frqFileOffset = $this->_frqFile->tell(); + + if ($this->_prxFile !== null) { + $this->_prxFile = null; + } + $this->_prxFile = $this->openCompoundFile('.prx', false); + $this->_prxFileOffset = $this->_prxFile->tell(); + + $this->_lastTerm = new Zend_Search_Lucene_Index_Term('', -1); + $this->_lastTermInfo = new Zend_Search_Lucene_Index_TermInfo(0, 0, 0, 0); + + $this->_docMap = array(); + for ($count = 0; $count < $this->_docCount; $count++) { + if (!$this->isDeleted($count)) { + $this->_docMap[$count] = $startId + ($compact ? count($this->_docMap) : $count); + } + } + + $this->nextTerm(); + return $startId + ($compact ? count($this->_docMap) : $this->_docCount); + } + + + /** + * Scans terms dictionary and returns next term + * + * @return Zend_Search_Lucene_Index_Term|null + */ + public function nextTerm() + { + if ($this->_tisFile === null || $this->_termCount == 0) { + $this->_lastTerm = null; + $this->_lastTermInfo = null; + + // may be necessary for "empty" segment + $this->_tisFile = null; + $this->_frqFile = null; + $this->_prxFile = null; + + return null; + } + + $termPrefixLength = $this->_tisFile->readVInt(); + $termSuffix = $this->_tisFile->readString(); + $termFieldNum = $this->_tisFile->readVInt(); + $termValue = Zend_Search_Lucene_Index_Term::getPrefix($this->_lastTerm->text, $termPrefixLength) . $termSuffix; + + $this->_lastTerm = new Zend_Search_Lucene_Index_Term($termValue, $this->_fields[$termFieldNum]->name); + + $docFreq = $this->_tisFile->readVInt(); + $freqPointer = $this->_lastTermInfo->freqPointer + $this->_tisFile->readVInt(); + $proxPointer = $this->_lastTermInfo->proxPointer + $this->_tisFile->readVInt(); + if ($docFreq >= $this->_skipInterval) { + $skipOffset = $this->_tisFile->readVInt(); + } else { + $skipOffset = 0; + } + + $this->_lastTermInfo = new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipOffset); + + + $this->_lastTermPositions = array(); + + $this->_frqFile->seek($this->_lastTermInfo->freqPointer + $this->_frqFileOffset, SEEK_SET); + $freqs = array(); $docId = 0; + for( $count = 0; $count < $this->_lastTermInfo->docFreq; $count++ ) { + $docDelta = $this->_frqFile->readVInt(); + if( $docDelta % 2 == 1 ) { + $docId += ($docDelta-1)/2; + $freqs[ $docId ] = 1; + } else { + $docId += $docDelta/2; + $freqs[ $docId ] = $this->_frqFile->readVInt(); + } + } + + $this->_prxFile->seek($this->_lastTermInfo->proxPointer + $this->_prxFileOffset, SEEK_SET); + foreach ($freqs as $docId => $freq) { + $termPosition = 0; $positions = array(); + + for ($count = 0; $count < $freq; $count++ ) { + $termPosition += $this->_prxFile->readVInt(); + $positions[] = $termPosition; + } + + if (isset($this->_docMap[$docId])) { + $this->_lastTermPositions[$this->_docMap[$docId]] = $positions; + } + } + + + $this->_termCount--; + if ($this->_termCount == 0) { + $this->_tisFile = null; + $this->_frqFile = null; + $this->_prxFile = null; + } + + return $this->_lastTerm; + } + + + /** + * Returns term in current position + * + * @param Zend_Search_Lucene_Index_Term $term + * @return Zend_Search_Lucene_Index_Term|null + */ + public function currentTerm() + { + return $this->_lastTerm; + } + + + /** + * Returns an array of all term positions in the documents. + * Return array structure: array( docId => array( pos1, pos2, ...), ...) + * + * @return array + */ + public function currentTermPositions() + { + return $this->_lastTermPositions; + } } diff --git a/search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php b/search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php new file mode 100644 index 00000000000..4d0f346ceb2 --- /dev/null +++ b/search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php @@ -0,0 +1,53 @@ +dirroot.'/search/Zend/Search/Lucene/Exception.php'; + +/** Zend_Search_Lucene */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/PriorityQueue.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Index + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Index_SegmentInfoPriorityQueue extends Zend_Search_Lucene_PriorityQueue +{ + /** + * Compare elements + * + * Returns true, if $el1 is less than $el2; else otherwise + * + * @param mixed $segmentInfo1 + * @param mixed $segmentInfo2 + * @return boolean + */ + protected function _less($segmentInfo1, $segmentInfo2) + { + return strcmp($segmentInfo1->currentTerm()->key(), $segmentInfo2->currentTerm()->key()) < 0; + } + +} diff --git a/search/Zend/Search/Lucene/Index/SegmentMerger.php b/search/Zend/Search/Lucene/Index/SegmentMerger.php new file mode 100644 index 00000000000..157489c492e --- /dev/null +++ b/search/Zend/Search/Lucene/Index/SegmentMerger.php @@ -0,0 +1,273 @@ +dirroot.'/search/Zend/Search/Lucene/Exception.php'; + +/** Zend_Search_Lucene_Index_SegmentInfo */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php'; + +/** Zend_Search_Lucene_Index_SegmentWriter_StreamWriter */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentWriter/StreamWriter.php'; + +/** Zend_Search_Lucene_Index_SegmentInfoPriorityQueue */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Index + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Index_SegmentMerger +{ + /** + * Target segment writer + * + * @var Zend_Search_Lucene_Index_SegmentWriter_StreamWriter + */ + private $_writer; + + /** + * Number of docs in a new segment + * + * @var integer + */ + private $_docCount; + + /** + * A set of segments to be merged + * + * @var array Zend_Search_Lucene_Index_SegmentInfo + */ + private $_segmentInfos = array(); + + /** + * Flag to signal, that merge is already done + * + * @var boolean + */ + private $_mergeDone = false; + + /** + * Field map + * [][] => + * + * @var array + */ + private $_fieldsMap = array(); + + + + /** + * Object constructor. + * + * Creates new segment merger with $directory as target to merge segments into + * and $name as a name of new segment + * + * @param Zend_Search_Lucene_Storage_Directory $directory + * @param string $name + */ + public function __construct($directory, $name) + { + $this->_writer = new Zend_Search_Lucene_Index_SegmentWriter_StreamWriter($directory, $name); + } + + + /** + * Add segmnet to a collection of segments to be merged + * + * @param Zend_Search_Lucene_Index_SegmentInfo $segment + */ + public function addSource(Zend_Search_Lucene_Index_SegmentInfo $segmentInfo) + { + $this->_segmentInfos[$segmentInfo->getName()] = $segmentInfo; + } + + + /** + * Do merge. + * + * Returns number of documents in newly created segment + * + * @return Zend_Search_Lucene_Index_SegmentInfo + * @throws Zend_Search_Lucene_Exception + */ + public function merge() + { + if ($this->_mergeDone) { + throw new Zend_Search_Lucene_Exception('Merge is already done.'); + } + + if (count($this->_segmentInfos) < 1) { + throw new Zend_Search_Lucene_Exception('Wrong number of segments to be merged (' + . count($this->_segmentInfos) + . ').'); + } + + $this->_mergeFields(); + $this->_mergeNorms(); + $this->_mergeStoredFields(); + $this->_mergeTerms(); + + $this->_mergeDone = true; + + return $this->_writer->close(); + } + + + /** + * Merge fields information + */ + private function _mergeFields() + { + foreach ($this->_segmentInfos as $segName => $segmentInfo) { + foreach ($segmentInfo->getFieldInfos() as $fieldInfo) { + $this->_fieldsMap[$segName][$fieldInfo->number] = $this->_writer->addFieldInfo($fieldInfo); + } + } + } + + /** + * Merge field's normalization factors + */ + private function _mergeNorms() + { + foreach ($this->_writer->getFieldInfos() as $fieldInfo) { + if ($fieldInfo->isIndexed) { + foreach ($this->_segmentInfos as $segName => $segmentInfo) { + if ($segmentInfo->hasDeletions()) { + $srcNorm = $segmentInfo->normVector($fieldInfo->name); + $norm = ''; + $docs = $segmentInfo->count(); + for ($count = 0; $count < $docs; $count++) { + if (!$segmentInfo->isDeleted($count)) { + $norm .= $srcNorm[$count]; + } + } + $this->_writer->addNorm($fieldInfo->name, $norm); + } else { + $this->_writer->addNorm($fieldInfo->name, $segmentInfo->normVector($fieldInfo->name)); + } + } + } + } + } + + /** + * Merge fields information + */ + private function _mergeStoredFields() + { + $this->_docCount = 0; + + foreach ($this->_segmentInfos as $segName => $segmentInfo) { + $fdtFile = $segmentInfo->openCompoundFile('.fdt'); + + for ($count = 0; $count < $segmentInfo->count(); $count++) { + $fieldCount = $fdtFile->readVInt(); + $storedFields = array(); + + for ($count2 = 0; $count2 < $fieldCount; $count2++) { + $fieldNum = $fdtFile->readVInt(); + $bits = $fdtFile->readByte(); + $fieldInfo = $segmentInfo->getField($fieldNum); + + if (!($bits & 2)) { // Text data + $storedFields[] = + new Zend_Search_Lucene_Field($fieldInfo->name, + $fdtFile->readString(), + 'UTF-8', + true, + $fieldInfo->isIndexed, + $bits & 1 ); + } else { // Binary data + $storedFields[] = + new Zend_Search_Lucene_Field($fieldInfo->name, + $fdtFile->readBinary(), + '', + true, + $fieldInfo->isIndexed, + $bits & 1, + true); + } + } + + if (!$segmentInfo->isDeleted($count)) { + $this->_docCount++; + $this->_writer->addStoredFields($storedFields); + } + } + } + } + + + /** + * Merge fields information + */ + private function _mergeTerms() + { + $segmentInfoQueue = new Zend_Search_Lucene_Index_SegmentInfoPriorityQueue(); + + $segmentStartId = 0; + foreach ($this->_segmentInfos as $segName => $segmentInfo) { + $segmentStartId = $segmentInfo->reset($segmentStartId, true); + + // Skip "empty" segments + if ($segmentInfo->currentTerm() !== null) { + $segmentInfoQueue->put($segmentInfo); + } + } + + $this->_writer->initializeDictionaryFiles(); + + $termDocs = array(); + while (($segmentInfo = $segmentInfoQueue->pop()) !== null) { + // Merge positions array + $termDocs += $segmentInfo->currentTermPositions(); + + if ($segmentInfoQueue->top() === null || + $segmentInfoQueue->top()->currentTerm()->key() != + $segmentInfo->currentTerm()->key()) { + // We got new term + ksort($termDocs, SORT_NUMERIC); + + // Add term if it's contained in any document + if (count($termDocs) > 0) { + $this->_writer->addTerm($segmentInfo->currentTerm(), $termDocs); + } + $termDocs = array(); + } + + $segmentInfo->nextTerm(); + // check, if segment dictionary is finished + if ($segmentInfo->currentTerm() !== null) { + // Put segment back into the priority queue + $segmentInfoQueue->put($segmentInfo); + } + } + + $this->_writer->closeDictionaryFiles(); + } +} diff --git a/search/Zend/Search/Lucene/Index/SegmentWriter.php b/search/Zend/Search/Lucene/Index/SegmentWriter.php index 6cb4477f99f..2f1a05e3229 100644 --- a/search/Zend/Search/Lucene/Index/SegmentWriter.php +++ b/search/Zend/Search/Lucene/Index/SegmentWriter.php @@ -15,29 +15,26 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Exception */ -require_once 'Zend/Search/Lucene/Exception.php'; - -/** Zend_Search_Lucene_Analysis_Analyzer */ -require_once 'Zend/Search/Lucene/Analysis/Analyzer.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; /** Zend_Search_Lucene_Index_SegmentInfo */ -require_once 'Zend/Search/Lucene/Index/SegmentInfo.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ -class Zend_Search_Lucene_Index_SegmentWriter +abstract class Zend_Search_Lucene_Index_SegmentWriter { /** * Expert: The fraction of terms in the "dictionary" which should be stored @@ -48,7 +45,7 @@ class Zend_Search_Lucene_Index_SegmentWriter * * @var integer */ - static public $indexInterval = 128; + public static $indexInterval = 128; /** Expert: The fraction of TermDocs entries stored in skip tables. * Larger values result in smaller indexes, greater acceleration, but fewer @@ -61,28 +58,28 @@ class Zend_Search_Lucene_Index_SegmentWriter * * @var integer */ - static public $skipInterval = 0x7FFFFFFF; + public static $skipInterval = 0x7FFFFFFF; /** * Number of docs in a segment * * @var integer */ - private $_docCount; + protected $_docCount = 0; /** * Segment name * * @var string */ - private $_name; + protected $_name; /** * File system adapter. * * @var Zend_Search_Lucene_Storage_Directory */ - private $_directory; + protected $_directory; /** * List of the index files. @@ -90,52 +87,41 @@ class Zend_Search_Lucene_Index_SegmentWriter * * @var unknown_type */ - private $_files; - - /** - * Term Dictionary - * Array of the Zend_Search_Lucene_Index_Term objects - * Corresponding Zend_Search_Lucene_Index_TermInfo object stored in the $_termDictionaryInfos - * - * @var array - */ - private $_termDictionary; - - /** - * Documents, which contain the term - * - * @var array - */ - private $_termDocs; + protected $_files = array(); /** * Segment fields. Array of Zend_Search_Lucene_Index_FieldInfo objects for this segment * * @var array */ - private $_fields; + protected $_fields = array(); /** - * Sizes of the indexed fields. - * Used for normalization factors calculation. + * Normalization factors. + * An array fieldName => normVector + * normVector is a binary string. + * Each byte corresponds to an indexed document in a segment and + * encodes normalization factor (float value, encoded by + * Zend_Search_Lucene_Search_Similarity::encodeNorm()) * * @var array */ - private $_fieldLengths; + protected $_norms = array(); + /** * '.fdx' file - Stored Fields, the field index. * * @var Zend_Search_Lucene_Storage_File */ - private $_fdxFile; + protected $_fdxFile = null; /** * '.fdt' file - Stored Fields, the field data. * * @var Zend_Search_Lucene_Storage_File */ - private $_fdtFile; + protected $_fdtFile = null; /** @@ -144,132 +130,125 @@ class Zend_Search_Lucene_Index_SegmentWriter * @param Zend_Search_Lucene_Storage_Directory $directory * @param string $name */ - public function __construct($directory, $name) + public function __construct(Zend_Search_Lucene_Storage_Directory $directory, $name) { $this->_directory = $directory; $this->_name = $name; - $this->_docCount = 0; - - $this->_fields = array(); - $this->_termDocs = array(); - $this->_files = array(); - $this->_norms = array(); - $this->_fieldLengths = array(); - $this->_termDictionary = array(); - - $this->_fdxFile = null; - $this->_fdtFile = null; } /** * Add field to the segment * + * Returns actual field number + * * @param Zend_Search_Lucene_Field $field + * @return integer */ - private function _addFieldInfo(Zend_Search_Lucene_Field $field) + public function addField(Zend_Search_Lucene_Field $field) { if (!isset($this->_fields[$field->name])) { + $fieldNumber = count($this->_fields); $this->_fields[$field->name] = new Zend_Search_Lucene_Index_FieldInfo($field->name, $field->isIndexed, - count($this->_fields), + $fieldNumber, $field->storeTermVector); + + return $fieldNumber; } else { $this->_fields[$field->name]->isIndexed |= $field->isIndexed; $this->_fields[$field->name]->storeTermVector |= $field->storeTermVector; + + return $this->_fields[$field->name]->number; } } + /** + * Add fieldInfo to the segment + * + * Returns actual field number + * + * @param Zend_Search_Lucene_Index_FieldInfo $fieldInfo + * @return integer + */ + public function addFieldInfo(Zend_Search_Lucene_Index_FieldInfo $fieldInfo) + { + if (!isset($this->_fields[$fieldInfo->name])) { + $fieldNumber = count($this->_fields); + $this->_fields[$fieldInfo->name] = + new Zend_Search_Lucene_Index_FieldInfo($fieldInfo->name, + $fieldInfo->isIndexed, + $fieldNumber, + $fieldInfo->storeTermVector); + + return $fieldNumber; + } else { + $this->_fields[$fieldInfo->name]->isIndexed |= $fieldInfo->isIndexed; + $this->_fields[$fieldInfo->name]->storeTermVector |= $fieldInfo->storeTermVector; + + return $this->_fields[$fieldInfo->name]->number; + } + } /** - * Adds a document to this segment. + * Returns array of FieldInfo objects. * - * @param Zend_Search_Lucene_Document $document - * @throws Zend_Search_Lucene_Exception + * @return array */ - public function addDocument(Zend_Search_Lucene_Document $document) + public function getFieldInfos() { - $storedFields = array(); + return $this->_fields; + } - foreach ($document->getFieldNames() as $fieldName) { - $field = $document->getField($fieldName); - $this->_addFieldInfo($field); + /** + * Add stored fields information + * + * @param array $storedFields array of Zend_Search_Lucene_Field objects + */ + public function addStoredFields($storedFields) + { + if (!isset($this->_fdxFile)) { + $this->_fdxFile = $this->_directory->createFile($this->_name . '.fdx'); + $this->_fdtFile = $this->_directory->createFile($this->_name . '.fdt'); - if ($field->storeTermVector) { - /** - * @todo term vector storing support - */ - throw new Zend_Search_Lucene_Exception('Store term vector functionality is not supported yet.'); - } - - if ($field->isIndexed) { - if ($field->isTokenized) { - $tokenList = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($field->stringValue); - } else { - $tokenList = array(); - $tokenList[] = new Zend_Search_Lucene_Analysis_Token($field->stringValue, 0, strlen($field->stringValue)); - } - $this->_fieldLengths[$field->name][$this->_docCount] = count($tokenList); - - $position = 0; - foreach ($tokenList as $token) { - $term = new Zend_Search_Lucene_Index_Term($token->getTermText(), $field->name); - $termKey = $term->key(); - - if (!isset($this->_termDictionary[$termKey])) { - // New term - $this->_termDictionary[$termKey] = $term; - $this->_termDocs[$termKey] = array(); - $this->_termDocs[$termKey][$this->_docCount] = array(); - } else if (!isset($this->_termDocs[$termKey][$this->_docCount])) { - // Existing term, but new term entry - $this->_termDocs[$termKey][$this->_docCount] = array(); - } - $position += $token->getPositionIncrement(); - $this->_termDocs[$termKey][$this->_docCount][] = $position; - } - } - - if ($field->isStored) { - $storedFields[] = $field; - } + $this->_files[] = $this->_name . '.fdx'; + $this->_files[] = $this->_name . '.fdt'; } - if (count($storedFields) != 0) { - if (!isset($this->_fdxFile)) { - $this->_fdxFile = $this->_directory->createFile($this->_name . '.fdx'); - $this->_fdtFile = $this->_directory->createFile($this->_name . '.fdt'); - - $this->_files[] = $this->_name . '.fdx'; - $this->_files[] = $this->_name . '.fdt'; - } - - $this->_fdxFile->writeLong($this->_fdtFile->tell()); - $this->_fdtFile->writeVInt(count($storedFields)); - foreach ($storedFields as $field) { - $this->_fdtFile->writeVInt($this->_fields[$field->name]->number); - $fieldBits = ($field->isTokenized ? 0x01 : 0x00) | - ($field->isBinary ? 0x02 : 0x00) | - 0x00; /* 0x04 - third bit, compressed (ZLIB) */ - $this->_fdtFile->writeByte($fieldBits); - if ($field->isBinary) { - $this->_fdtFile->writeVInt(strlen($field->stringValue)); - $this->_fdtFile->writeBytes($field->stringValue); - } else { - $this->_fdtFile->writeString($field->stringValue); - } + $this->_fdxFile->writeLong($this->_fdtFile->tell()); + $this->_fdtFile->writeVInt(count($storedFields)); + foreach ($storedFields as $field) { + $this->_fdtFile->writeVInt($this->_fields[$field->name]->number); + $fieldBits = ($field->isTokenized ? 0x01 : 0x00) | + ($field->isBinary ? 0x02 : 0x00) | + 0x00; /* 0x04 - third bit, compressed (ZLIB) */ + $this->_fdtFile->writeByte($fieldBits); + if ($field->isBinary) { + $this->_fdtFile->writeVInt(strlen($field->value)); + $this->_fdtFile->writeBytes($field->value); + } else { + $this->_fdtFile->writeString($field->getUtf8Value()); } } $this->_docCount++; } + /** + * Returns the total number of documents in this segment. + * + * @return integer + */ + public function count() + { + return $this->_docCount; + } /** * Dump Field Info (.fnm) segment file */ - private function _dumpFNM() + protected function _dumpFNM() { $fnmFile = $this->_directory->createFile($this->_name . '.fnm'); $fnmFile->writeVInt(count($this->_fields)); @@ -283,20 +262,9 @@ class Zend_Search_Lucene_Index_SegmentWriter ); if ($field->isIndexed) { - $fieldNum = $this->_fields[$field->name]->number; - $fieldName = $field->name; - $similarity = Zend_Search_Lucene_Search_Similarity::getDefault(); - $norm = ''; - - for ($count = 0; $count < $this->_docCount; $count++) { - $numTokens = isset($this->_fieldLengths[$fieldName][$count]) ? - $this->_fieldLengths[$fieldName][$count] : 0; - $norm .= chr($similarity->encodeNorm($similarity->lengthNorm($fieldName, $numTokens))); - } - - $normFileName = $this->_name . '.f' . $fieldNum; + $normFileName = $this->_name . '.f' . $field->number; $fFile = $this->_directory->createFile($normFileName); - $fFile->writeBytes($norm); + $fFile->writeBytes($this->_norms[$field->name]); $this->_files[] = $normFileName; } } @@ -305,6 +273,194 @@ class Zend_Search_Lucene_Index_SegmentWriter } + + /** + * Term Dictionary file + * + * @var Zend_Search_Lucene_Storage_File + */ + private $_tisFile = null; + + /** + * Term Dictionary index file + * + * @var Zend_Search_Lucene_Storage_File + */ + private $_tiiFile = null; + + /** + * Frequencies file + * + * @var Zend_Search_Lucene_Storage_File + */ + private $_frqFile = null; + + /** + * Positions file + * + * @var Zend_Search_Lucene_Storage_File + */ + private $_prxFile = null; + + /** + * Number of written terms + * + * @var integer + */ + private $_termCount; + + + /** + * Last saved term + * + * @var Zend_Search_Lucene_Index_Term + */ + private $_prevTerm; + + /** + * Last saved term info + * + * @var Zend_Search_Lucene_Index_TermInfo + */ + private $_prevTermInfo; + + /** + * Last saved index term + * + * @var Zend_Search_Lucene_Index_Term + */ + private $_prevIndexTerm; + + /** + * Last saved index term info + * + * @var Zend_Search_Lucene_Index_TermInfo + */ + private $_prevIndexTermInfo; + + /** + * Last term dictionary file position + * + * @var integer + */ + private $_lastIndexPosition; + + /** + * Create dicrionary, frequency and positions files and write necessary headers + */ + public function initializeDictionaryFiles() + { + $this->_tisFile = $this->_directory->createFile($this->_name . '.tis'); + $this->_tisFile->writeInt((int)0xFFFFFFFE); + $this->_tisFile->writeLong(0 /* dummy data for terms count */); + $this->_tisFile->writeInt(self::$indexInterval); + $this->_tisFile->writeInt(self::$skipInterval); + + $this->_tiiFile = $this->_directory->createFile($this->_name . '.tii'); + $this->_tiiFile->writeInt((int)0xFFFFFFFE); + $this->_tiiFile->writeLong(0 /* dummy data for terms count */); + $this->_tiiFile->writeInt(self::$indexInterval); + $this->_tiiFile->writeInt(self::$skipInterval); + + /** Dump dictionary header */ + $this->_tiiFile->writeVInt(0); // preffix length + $this->_tiiFile->writeString(''); // suffix + $this->_tiiFile->writeInt((int)0xFFFFFFFF); // field number + $this->_tiiFile->writeByte((int)0x0F); + $this->_tiiFile->writeVInt(0); // DocFreq + $this->_tiiFile->writeVInt(0); // FreqDelta + $this->_tiiFile->writeVInt(0); // ProxDelta + $this->_tiiFile->writeVInt(20); // IndexDelta + + $this->_frqFile = $this->_directory->createFile($this->_name . '.frq'); + $this->_prxFile = $this->_directory->createFile($this->_name . '.prx'); + + $this->_files[] = $this->_name . '.tis'; + $this->_files[] = $this->_name . '.tii'; + $this->_files[] = $this->_name . '.frq'; + $this->_files[] = $this->_name . '.prx'; + + $this->_prevTerm = null; + $this->_prevTermInfo = null; + $this->_prevIndexTerm = null; + $this->_prevIndexTermInfo = null; + $this->_lastIndexPosition = 20; + $this->_termCount = 0; + + } + + /** + * Add term + * + * Term positions is an array( docId => array(pos1, pos2, pos3, ...), ... ) + * + * @param Zend_Search_Lucene_Index_Term $termEntry + * @param array $termDocs + */ + public function addTerm($termEntry, $termDocs) + { + $freqPointer = $this->_frqFile->tell(); + $proxPointer = $this->_prxFile->tell(); + + $prevDoc = 0; + foreach ($termDocs as $docId => $termPositions) { + $docDelta = ($docId - $prevDoc)*2; + $prevDoc = $docId; + if (count($termPositions) > 1) { + $this->_frqFile->writeVInt($docDelta); + $this->_frqFile->writeVInt(count($termPositions)); + } else { + $this->_frqFile->writeVInt($docDelta + 1); + } + + $prevPosition = 0; + foreach ($termPositions as $position) { + $this->_prxFile->writeVInt($position - $prevPosition); + $prevPosition = $position; + } + } + + if (count($termDocs) >= self::$skipInterval) { + /** + * @todo Write Skip Data to a freq file. + * It's not used now, but make index more optimal + */ + $skipOffset = $this->_frqFile->tell() - $freqPointer; + } else { + $skipOffset = 0; + } + + $term = new Zend_Search_Lucene_Index_Term($termEntry->text, + $this->_fields[$termEntry->field]->number); + $termInfo = new Zend_Search_Lucene_Index_TermInfo(count($termDocs), + $freqPointer, $proxPointer, $skipOffset); + + $this->_dumpTermDictEntry($this->_tisFile, $this->_prevTerm, $term, $this->_prevTermInfo, $termInfo); + + if (($this->_termCount + 1) % self::$indexInterval == 0) { + $this->_dumpTermDictEntry($this->_tiiFile, $this->_prevIndexTerm, $term, $this->_prevIndexTermInfo, $termInfo); + + $indexPosition = $this->_tisFile->tell(); + $this->_tiiFile->writeVInt($indexPosition - $this->_lastIndexPosition); + $this->_lastIndexPosition = $indexPosition; + + } + $this->_termCount++; + } + + /** + * Close dictionary + */ + public function closeDictionaryFiles() + { + $this->_tisFile->seek(4); + $this->_tisFile->writeLong($this->_termCount); + + $this->_tiiFile->seek(4); + $this->_tiiFile->writeLong(ceil(($this->_termCount + 2)/self::$indexInterval)); + } + + /** * Dump Term Dictionary segment file entry. * Used to write entry to .tis or .tii files @@ -315,22 +471,47 @@ class Zend_Search_Lucene_Index_SegmentWriter * @param Zend_Search_Lucene_Index_TermInfo $prevTermInfo * @param Zend_Search_Lucene_Index_TermInfo $termInfo */ - private function _dumpTermDictEntry(Zend_Search_Lucene_Storage_File $dicFile, + protected function _dumpTermDictEntry(Zend_Search_Lucene_Storage_File $dicFile, &$prevTerm, Zend_Search_Lucene_Index_Term $term, &$prevTermInfo, Zend_Search_Lucene_Index_TermInfo $termInfo) { if (isset($prevTerm) && $prevTerm->field == $term->field) { - $prefixLength = 0; - while ($prefixLength < strlen($prevTerm->text) && - $prefixLength < strlen($term->text) && - $prevTerm->text{$prefixLength} == $term->text{$prefixLength} - ) { - $prefixLength++; + $matchedBytes = 0; + $maxBytes = min(strlen($prevTerm->text), strlen($term->text)); + while ($matchedBytes < $maxBytes && + $prevTerm->text[$matchedBytes] == $term->text[$matchedBytes]) { + $matchedBytes++; } + + // Calculate actual matched UTF-8 pattern + $prefixBytes = 0; + $prefixChars = 0; + while ($prefixBytes < $matchedBytes) { + $charBytes = 1; + if ((ord($term->text[$prefixBytes]) & 0xC0) == 0xC0) { + $charBytes++; + if (ord($term->text[$prefixBytes]) & 0x20 ) { + $charBytes++; + if (ord($term->text[$prefixBytes]) & 0x10 ) { + $charBytes++; + } + } + } + + if ($prefixBytes + $charBytes > $matchedBytes) { + // char crosses matched bytes boundary + // skip char + break; + } + + $prefixChars++; + $prefixBytes += $charBytes; + } + // Write preffix length - $dicFile->writeVInt($prefixLength); + $dicFile->writeVInt($prefixChars); // Write suffix - $dicFile->writeString( substr($term->text, $prefixLength) ); + $dicFile->writeString(substr($term->text, $prefixBytes)); } else { // Write preffix length $dicFile->writeVInt(0); @@ -363,107 +544,11 @@ class Zend_Search_Lucene_Index_SegmentWriter $prevTermInfo = $termInfo; } - /** - * Dump Term Dictionary (.tis) and Term Dictionary Index (.tii) segment files - */ - private function _dumpDictionary() - { - $termKeys = array_keys($this->_termDictionary); - sort($termKeys, SORT_STRING); - - $tisFile = $this->_directory->createFile($this->_name . '.tis'); - $tisFile->writeInt((int)0xFFFFFFFE); - $tisFile->writeLong(count($termKeys)); - $tisFile->writeInt(self::$indexInterval); - $tisFile->writeInt(self::$skipInterval); - - $tiiFile = $this->_directory->createFile($this->_name . '.tii'); - $tiiFile->writeInt((int)0xFFFFFFFE); - $tiiFile->writeLong(ceil((count($termKeys) + 2)/self::$indexInterval)); - $tiiFile->writeInt(self::$indexInterval); - $tiiFile->writeInt(self::$skipInterval); - - /** Dump dictionary header */ - $tiiFile->writeVInt(0); // preffix length - $tiiFile->writeString(''); // suffix - $tiiFile->writeInt((int)0xFFFFFFFF); // field number - $tiiFile->writeByte((int)0x0F); - $tiiFile->writeVInt(0); // DocFreq - $tiiFile->writeVInt(0); // FreqDelta - $tiiFile->writeVInt(0); // ProxDelta - $tiiFile->writeVInt(20); // IndexDelta - - $frqFile = $this->_directory->createFile($this->_name . '.frq'); - $prxFile = $this->_directory->createFile($this->_name . '.prx'); - - $termCount = 1; - - $prevTerm = null; - $prevTermInfo = null; - $prevIndexTerm = null; - $prevIndexTermInfo = null; - $prevIndexPosition = 20; - - foreach ($termKeys as $termId) { - $freqPointer = $frqFile->tell(); - $proxPointer = $prxFile->tell(); - - $prevDoc = 0; - foreach ($this->_termDocs[$termId] as $docId => $termPositions) { - $docDelta = ($docId - $prevDoc)*2; - $prevDoc = $docId; - if (count($termPositions) > 1) { - $frqFile->writeVInt($docDelta); - $frqFile->writeVInt(count($termPositions)); - } else { - $frqFile->writeVInt($docDelta + 1); - } - - $prevPosition = 0; - foreach ($termPositions as $position) { - $prxFile->writeVInt($position - $prevPosition); - $prevPosition = $position; - } - } - - if (count($this->_termDocs[$termId]) >= self::$skipInterval) { - /** - * @todo Write Skip Data to a freq file. - * It's not used now, but make index more optimal - */ - $skipOffset = $frqFile->tell() - $freqPointer; - } else { - $skipOffset = 0; - } - - $term = new Zend_Search_Lucene_Index_Term($this->_termDictionary[$termId]->text, - $this->_fields[$this->_termDictionary[$termId]->field]->number); - $termInfo = new Zend_Search_Lucene_Index_TermInfo(count($this->_termDocs[$termId]), - $freqPointer, $proxPointer, $skipOffset); - - $this->_dumpTermDictEntry($tisFile, $prevTerm, $term, $prevTermInfo, $termInfo); - - if ($termCount % self::$indexInterval == 0) { - $this->_dumpTermDictEntry($tiiFile, $prevIndexTerm, $term, $prevIndexTermInfo, $termInfo); - - $indexPosition = $tisFile->tell(); - $tiiFile->writeVInt($indexPosition - $prevIndexPosition); - $prevIndexPosition = $indexPosition; - } - $termCount++; - } - - $this->_files[] = $this->_name . '.tis'; - $this->_files[] = $this->_name . '.tii'; - $this->_files[] = $this->_name . '.frq'; - $this->_files[] = $this->_name . '.prx'; - } - /** * Generate compound index file */ - private function _generateCFS() + protected function _generateCFS() { $cfsFile = $this->_directory->createFile($this->_name . '.cfs'); $cfsFile->writeVInt(count($this->_files)); @@ -486,8 +571,13 @@ class Zend_Search_Lucene_Index_SegmentWriter $cfsFile->seek($dataOffset); $dataFile = $this->_directory->getFileObject($fileName); - $data = $dataFile->readBytes($this->_directory->fileLength($fileName)); - $cfsFile->writeBytes($data); + + $byteCount = $this->_directory->fileLength($fileName); + while ($byteCount > 0) { + $data = $dataFile->readBytes(min($byteCount, 131072 /*128Kb*/)); + $byteCount -= strlen($data); + $cfsFile->writeBytes($data); + } $this->_directory->deleteFile($fileName); } @@ -499,21 +589,6 @@ class Zend_Search_Lucene_Index_SegmentWriter * * @return Zend_Search_Lucene_Index_SegmentInfo */ - public function close() - { - if ($this->_docCount == 0) { - return null; - } - - $this->_dumpFNM(); - $this->_dumpDictionary(); - - $this->_generateCFS(); - - return new Zend_Search_Lucene_Index_SegmentInfo($this->_name, - $this->_docCount, - $this->_directory); - } - + abstract public function close(); } diff --git a/search/Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php b/search/Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php new file mode 100644 index 00000000000..7dd2bf91587 --- /dev/null +++ b/search/Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php @@ -0,0 +1,213 @@ +dirroot.'/search/Zend/Search/Lucene/Exception.php'; + +/** Zend_Search_Lucene_Analysis_Analyzer */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php'; + +/** Zend_Search_Lucene_Index_SegmentWriter */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentWriter.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Index + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter extends Zend_Search_Lucene_Index_SegmentWriter +{ + /** + * Term Dictionary + * Array of the Zend_Search_Lucene_Index_Term objects + * Corresponding Zend_Search_Lucene_Index_TermInfo object stored in the $_termDictionaryInfos + * + * @var array + */ + protected $_termDictionary; + + /** + * Documents, which contain the term + * + * @var array + */ + protected $_termDocs; + + /** + * Object constructor. + * + * @param Zend_Search_Lucene_Storage_Directory $directory + * @param string $name + */ + public function __construct(Zend_Search_Lucene_Storage_Directory $directory, $name) + { + parent::__construct($directory, $name); + + $this->_termDocs = array(); + $this->_termDictionary = array(); + } + + + /** + * Adds a document to this segment. + * + * @param Zend_Search_Lucene_Document $document + * @throws Zend_Search_Lucene_Exception + */ + public function addDocument(Zend_Search_Lucene_Document $document) + { + $storedFields = array(); + $docNorms = array(); + $similarity = Zend_Search_Lucene_Search_Similarity::getDefault(); + + foreach ($document->getFieldNames() as $fieldName) { + $field = $document->getField($fieldName); + $this->addField($field); + + if ($field->storeTermVector) { + /** + * @todo term vector storing support + */ + throw new Zend_Search_Lucene_Exception('Store term vector functionality is not supported yet.'); + } + + if ($field->isIndexed) { + if ($field->isTokenized) { + $analyzer = Zend_Search_Lucene_Analysis_Analyzer::getDefault(); + $analyzer->setInput($field->value, $field->encoding); + + $position = 0; + $tokenCounter = 0; + while (($token = $analyzer->nextToken()) !== null) { + $tokenCounter++; + + $term = new Zend_Search_Lucene_Index_Term($token->getTermText(), $field->name); + $termKey = $term->key(); + + if (!isset($this->_termDictionary[$termKey])) { + // New term + $this->_termDictionary[$termKey] = $term; + $this->_termDocs[$termKey] = array(); + $this->_termDocs[$termKey][$this->_docCount] = array(); + } else if (!isset($this->_termDocs[$termKey][$this->_docCount])) { + // Existing term, but new term entry + $this->_termDocs[$termKey][$this->_docCount] = array(); + } + $position += $token->getPositionIncrement(); + $this->_termDocs[$termKey][$this->_docCount][] = $position; + } + + $docNorms[$field->name] = chr($similarity->encodeNorm( $similarity->lengthNorm($field->name, + $tokenCounter)* + $document->boost* + $field->boost )); + } else { + $term = new Zend_Search_Lucene_Index_Term($field->getUtf8Value(), $field->name); + $termKey = $term->key(); + + if (!isset($this->_termDictionary[$termKey])) { + // New term + $this->_termDictionary[$termKey] = $term; + $this->_termDocs[$termKey] = array(); + $this->_termDocs[$termKey][$this->_docCount] = array(); + } else if (!isset($this->_termDocs[$termKey][$this->_docCount])) { + // Existing term, but new term entry + $this->_termDocs[$termKey][$this->_docCount] = array(); + } + $this->_termDocs[$termKey][$this->_docCount][] = 0; // position + + $docNorms[$field->name] = chr($similarity->encodeNorm( $similarity->lengthNorm($field->name, 1)* + $document->boost* + $field->boost )); + } + } + + if ($field->isStored) { + $storedFields[] = $field; + } + } + + + foreach ($this->_fields as $fieldName => $field) { + if (!$field->isIndexed) { + continue; + } + + if (!isset($this->_norms[$fieldName])) { + $this->_norms[$fieldName] = str_repeat(chr($similarity->encodeNorm( $similarity->lengthNorm($fieldName, 0) )), + $this->_docCount); + } + + if (isset($docNorms[$fieldName])){ + $this->_norms[$fieldName] .= $docNorms[$fieldName]; + } else { + $this->_norms[$fieldName] .= chr($similarity->encodeNorm( $similarity->lengthNorm($fieldName, 0) )); + } + } + + $this->addStoredFields($storedFields); + } + + + /** + * Dump Term Dictionary (.tis) and Term Dictionary Index (.tii) segment files + */ + protected function _dumpDictionary() + { + ksort($this->_termDictionary, SORT_STRING); + + $this->initializeDictionaryFiles(); + + foreach ($this->_termDictionary as $termId => $term) { + $this->addTerm($term, $this->_termDocs[$termId]); + } + + $this->closeDictionaryFiles(); + } + + + /** + * Close segment, write it to disk and return segment info + * + * @return Zend_Search_Lucene_Index_SegmentInfo + */ + public function close() + { + if ($this->_docCount == 0) { + return null; + } + + $this->_dumpFNM(); + $this->_dumpDictionary(); + + $this->_generateCFS(); + + return new Zend_Search_Lucene_Index_SegmentInfo($this->_name, + $this->_docCount, + $this->_directory); + } + +} + diff --git a/search/Zend/Search/Lucene/Index/SegmentWriter/StreamWriter.php b/search/Zend/Search/Lucene/Index/SegmentWriter/StreamWriter.php new file mode 100644 index 00000000000..ba0e202522a --- /dev/null +++ b/search/Zend/Search/Lucene/Index/SegmentWriter/StreamWriter.php @@ -0,0 +1,94 @@ +dirroot.'/search/Zend/Search/Lucene/Exception.php'; + +/** Zend_Search_Lucene_Index_SegmentInfo */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php'; + +/** Zend_Search_Lucene_Index_SegmentWriter */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentWriter.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Index + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Index_SegmentWriter_StreamWriter extends Zend_Search_Lucene_Index_SegmentWriter +{ + /** + * Object constructor. + * + * @param Zend_Search_Lucene_Storage_Directory $directory + * @param string $name + */ + public function __construct(Zend_Search_Lucene_Storage_Directory $directory, $name) + { + parent::__construct($directory, $name); + } + + + /** + * Create stored fields files and open them for write + */ + public function createStoredFieldsFiles() + { + $this->_fdxFile = $this->_directory->createFile($this->_name . '.fdx'); + $this->_fdtFile = $this->_directory->createFile($this->_name . '.fdt'); + + $this->_files[] = $this->_name . '.fdx'; + $this->_files[] = $this->_name . '.fdt'; + } + + public function addNorm($fieldName, $normVector) + { + if (isset($this->_norms[$fieldName])) { + $this->_norms[$fieldName] .= $normVector; + } else { + $this->_norms[$fieldName] = $normVector; + } + } + + /** + * Close segment, write it to disk and return segment info + * + * @return Zend_Search_Lucene_Index_SegmentInfo + */ + public function close() + { + if ($this->_docCount == 0) { + return null; + } + + $this->_dumpFNM(); + $this->_generateCFS(); + + return new Zend_Search_Lucene_Index_SegmentInfo($this->_name, + $this->_docCount, + $this->_directory); + } +} + diff --git a/search/Zend/Search/Lucene/Index/Term.php b/search/Zend/Search/Lucene/Index/Term.php index 3deffa90ab2..465b4ef41ba 100644 --- a/search/Zend/Search/Lucene/Index/Term.php +++ b/search/Zend/Search/Lucene/Index/Term.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -31,7 +31,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_Term @@ -52,21 +52,57 @@ class Zend_Search_Lucene_Index_Term /** - * @todo docblock + * Object constructor */ - public function __construct( $text, $field = 'contents' ) + public function __construct($text, $field = null) { - $this->field = $field; - $this->text = $text; + $this->field = ($field === null)? Zend_Search_Lucene::getDefaultSearchField() : $field; + $this->text = $text; } /** - * @todo docblock + * Returns term key + * + * @return string */ public function key() { return $this->field . chr(0) . $this->text; } + + /** + * Get term prefix + * + * @param integer $length + * @return string + */ + public static function getPrefix($str, $length) + { + $prefixBytes = 0; + $prefixChars = 0; + while ($prefixBytes < strlen($str) && $prefixChars < $length) { + $charBytes = 1; + if ((ord($str[$prefixBytes]) & 0xC0) == 0xC0) { + $charBytes++; + if (ord($str[$prefixBytes]) & 0x20 ) { + $charBytes++; + if (ord($str[$prefixBytes]) & 0x10 ) { + $charBytes++; + } + } + } + + if ($prefixBytes + $charBytes > strlen($str)) { + // wrong character + break; + } + + $prefixChars++; + $prefixBytes += $charBytes; + } + + return substr($str, 0, $prefixBytes); + } } diff --git a/search/Zend/Search/Lucene/Index/TermInfo.php b/search/Zend/Search/Lucene/Index/TermInfo.php index 7dcfcc8a967..95f7cfc65f4 100644 --- a/search/Zend/Search/Lucene/Index/TermInfo.php +++ b/search/Zend/Search/Lucene/Index/TermInfo.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -26,7 +26,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_TermInfo diff --git a/search/Zend/Search/Lucene/Index/Writer.php b/search/Zend/Search/Lucene/Index/Writer.php index ef6c65526a3..8e32f4e1f6e 100644 --- a/search/Zend/Search/Lucene/Index/Writer.php +++ b/search/Zend/Search/Lucene/Index/Writer.php @@ -15,36 +15,80 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ -/** Zend_Search_Lucene_Index_SegmentWriter */ -require_once 'Zend/Search/Lucene/Index/SegmentWriter.php'; +/** Zend_Search_Lucene_Index_SegmentWriter_ */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php'; /** Zend_Search_Lucene_Index_SegmentInfo */ -require_once 'Zend/Search/Lucene/Index/SegmentInfo.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php'; + +/** Zend_Search_Lucene_Index_SegmentMerger */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentMerger.php'; + /** * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_Writer { /** - * @todo Implement segment merger - * @todo Implement mergeFactor, minMergeDocs, maxMergeDocs usage. * @todo Implement Analyzer substitution * @todo Implement Zend_Search_Lucene_Storage_DirectoryRAM and Zend_Search_Lucene_Storage_FileRAM to use it for * temporary index files * @todo Directory lock processing */ + /** + * Number of documents required before the buffered in-memory + * documents are written into a new Segment + * + * Default value is 10 + * + * @var integer + */ + public $maxBufferedDocs = 10; + + /** + * Largest number of documents ever merged by addDocument(). + * Small values (e.g., less than 10,000) are best for interactive indexing, + * as this limits the length of pauses while indexing to a few seconds. + * Larger values are best for batched indexing and speedier searches. + * + * Default value is PHP_INT_MAX + * + * @var integer + */ + public $maxMergeDocs = PHP_INT_MAX; + + /** + * Determines how often segment indices are merged by addDocument(). + * + * With smaller values, less RAM is used while indexing, + * and searches on unoptimized indices are faster, + * but indexing speed is slower. + * + * With larger values, more RAM is used during indexing, + * and while searches on unoptimized indices are slower, + * indexing is faster. + * + * Thus larger values (> 10) are best for batch index creation, + * and smaller values (< 10) for indices that are interactively maintained. + * + * Default value is 10 + * + * @var integer + */ + public $mergeFactor = 10; + /** * File system adapter. * @@ -54,51 +98,11 @@ class Zend_Search_Lucene_Index_Writer /** - * Index version - * Counts how often the index has been changed by adding or deleting docs + * Changes counter. * * @var integer */ - private $_version; - - /** - * Segment name counter. - * Used to name new segments . - * - * @var integer - */ - private $_segmentNameCounter; - - /** - * Number of the segments in the index - * - * @var inteher - */ - private $_segments; - - /** - * Determines how often segment indices - * are merged by addDocument(). - * - * @var integer - */ - public $mergeFactor; - - /** - * Determines the minimal number of documents required before - * the buffered in-memory documents are merging and a new Segment - * is created. - * - * @var integer - */ - public $minMergeDocs; - - /** - * Determines the largest number of documents ever merged by addDocument(). - * - * @var integer - */ - public $maxMergeDocs; + private $_versionUpdate = 0; /** * List of the segments, created by index writer @@ -106,14 +110,30 @@ class Zend_Search_Lucene_Index_Writer * * @var array */ - private $_newSegments; + private $_newSegments = array(); + + /** + * List of segments to be deleted on commit + * + * @var array + */ + private $_segmentsToDelete = array(); /** * Current segment to add documents * - * @var Zend_Search_Lucene_Index_SegmentWriter + * @var Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter */ - private $_currentSegment; + private $_currentSegment = null; + + /** + * Array of Zend_Search_Lucene_Index_SegmentInfo objects for this index. + * + * It's a reference to the corresponding Zend_Search_Lucene::$_segmentInfos array + * + * @var array Zend_Search_Lucene_Index_SegmentInfo + */ + private $_segmentInfos; /** * List of indexfiles extensions @@ -131,7 +151,8 @@ class Zend_Search_Lucene_Index_Writer '.tvx' => '.tvx', '.tvd' => '.tvd', '.tvf' => '.tvf', - '.del' => '.del' ); + '.del' => '.del', + '.sti' => '.sti' ); /** * Opens the index for writing @@ -142,11 +163,13 @@ class Zend_Search_Lucene_Index_Writer * index or overwrite the existing one. * * @param Zend_Search_Lucene_Storage_Directory $directory + * @param array $segmentInfos * @param boolean $create */ - public function __construct(Zend_Search_Lucene_Storage_Directory $directory, $create = false) + public function __construct(Zend_Search_Lucene_Storage_Directory $directory, &$segmentInfos, $create = false) { - $this->_directory = $directory; + $this->_directory = $directory; + $this->_segmentInfos = &$segmentInfos; if ($create) { foreach ($this->_directory->fileList() as $file) { @@ -159,8 +182,13 @@ class Zend_Search_Lucene_Index_Writer } $segmentsFile = $this->_directory->createFile('segments'); $segmentsFile->writeInt((int)0xFFFFFFFF); - // write version - $segmentsFile->writeLong(0); + + // write version (is initialized by current time + // $segmentsFile->writeLong((int)microtime(true)); + $version = microtime(true); + $segmentsFile->writeInt((int)($version/((double)0xFFFFFFFF + 1))); + $segmentsFile->writeInt((int)($version & 0xFFFFFFFF)); + // write name counter $segmentsFile->writeInt(0); // write segment counter @@ -169,27 +197,13 @@ class Zend_Search_Lucene_Index_Writer $deletableFile = $this->_directory->createFile('deletable'); // write counter $deletableFile->writeInt(0); - - $this->_version = 0; - $this->_segmentNameCounter = 0; - $this->_segments = 0; } else { $segmentsFile = $this->_directory->getFileObject('segments'); $format = $segmentsFile->readInt(); if ($format != (int)0xFFFFFFFF) { throw new Zend_Search_Lucene_Exception('Wrong segments file format'); } - - // read version - $this->_version = $segmentsFile->readLong(); - // read counter - $this->_segmentNameCounter = $segmentsFile->readInt(); - // read segment counter - $this->_segments = $segmentsFile->readInt(); } - - $this->_newSegments = array(); - $this->_currentSegment = null; } /** @@ -201,49 +215,218 @@ class Zend_Search_Lucene_Index_Writer { if ($this->_currentSegment === null) { $this->_currentSegment = - new Zend_Search_Lucene_Index_SegmentWriter($this->_directory, $this->_newSegmentName()); + new Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter($this->_directory, $this->_newSegmentName()); } $this->_currentSegment->addDocument($document); - $this->_version++; + + if ($this->_currentSegment->count() >= $this->maxBufferedDocs) { + $this->commit(); + } + + $this->_versionUpdate++; + + $this->_maybeMergeSegments(); } + /** + * Merge segments if necessary + */ + private function _maybeMergeSegments() + { + $segmentSizes = array(); + foreach ($this->_segmentInfos as $segId => $segmentInfo) { + $segmentSizes[$segId] = $segmentInfo->count(); + } + + $mergePool = array(); + $poolSize = 0; + $sizeToMerge = $this->maxBufferedDocs; + asort($segmentSizes, SORT_NUMERIC); + foreach ($segmentSizes as $segId => $size) { + // Check, if segment comes into a new merging block + while ($size >= $sizeToMerge) { + // Merge previous block if it's large enough + if ($poolSize >= $sizeToMerge) { + $this->_mergeSegments($mergePool); + } + $mergePool = array(); + $poolSize = 0; + + $sizeToMerge *= $this->mergeFactor; + + if ($sizeToMerge > $this->maxMergeDocs) { + return; + } + } + + $mergePool[] = $this->_segmentInfos[$segId]; + $poolSize += $size; + } + + if ($poolSize >= $sizeToMerge) { + $this->_mergeSegments($mergePool); + } + } + + /** + * Merge specified segments + * + * $segments is an array of SegmentInfo objects + * + * @param array $segments + */ + private function _mergeSegments($segments) + { + // Try to get exclusive non-blocking lock to the 'index.optimization.lock' + // Skip optimization if it's performed by other process right now + $optimizationLock = $this->_directory->createFile('index.optimization.lock'); + if (!$optimizationLock->lock(LOCK_EX,true)) { + return; + } + + $newName = $this->_newSegmentName(); + $merger = new Zend_Search_Lucene_Index_SegmentMerger($this->_directory, + $newName); + foreach ($segments as $segmentInfo) { + $merger->addSource($segmentInfo); + $this->_segmentsToDelete[$segmentInfo->getName()] = $segmentInfo->getName(); + } + + $newSegment = $merger->merge(); + if ($newSegment !== null) { + $this->_newSegments[$newSegment->getName()] = $newSegment; + } + + $this->commit(); + + // optimization is finished + $optimizationLock->unlock(); + } /** * Update segments file by adding current segment to a list - * @todo !!!!!Finish the implementation * * @throws Zend_Search_Lucene_Exception */ private function _updateSegments() { - $segmentsFile = $this->_directory->getFileObject('segments'); - $newSegmentFile = $this->_directory->createFile('segments.new'); - - $newSegmentFile->writeInt((int)0xFFFFFFFF); - $newSegmentFile->writeLong($this->_version); - $newSegmentFile->writeInt($this->_segmentNameCounter); - - $this->_segments += count($this->_newSegments); - $newSegmentFile->writeInt($this->_segments); - - $segmentsFile->seek(20); - $newSegmentFile->writeBytes($segmentsFile->readBytes($this->_directory->fileLength('segments') - 20)); - - foreach ($this->_newSegments as $segmentName => $segmentInfo) { - $newSegmentFile->writeString($segmentName); - $newSegmentFile->writeInt($segmentInfo->count()); + // Get an exclusive index lock + // Wait, until all parallel searchers or indexers won't stop + // and stop all next searchers, while we are updating segments file + $lock = $this->_directory->getFileObject('index.lock'); + if (!$lock->lock(LOCK_EX)) { + throw new Zend_Search_Lucene_Exception('Can\'t obtain exclusive index lock'); } + + // Do not share file handlers to get file updates from other sessions. + $segmentsFile = $this->_directory->getFileObject('segments', false); + $newSegmentFile = $this->_directory->createFile('segments.new', false); + + // Write format marker + $newSegmentFile->writeInt((int)0xFFFFFFFF); + + // Write index version + $segmentsFile->seek(4, SEEK_CUR); + // $version = $segmentsFile->readLong() + $this->_versionUpdate; + // Process version on 32-bit platforms + $versionHigh = $segmentsFile->readInt(); + $versionLow = $segmentsFile->readInt(); + $version = $versionHigh * ((double)0xFFFFFFFF + 1) + + (($versionLow < 0)? (double)0xFFFFFFFF - (-1 - $versionLow) : $versionLow); + $version += $this->_versionUpdate; + $this->_versionUpdate = 0; + $newSegmentFile->writeInt((int)($version/((double)0xFFFFFFFF + 1))); + $newSegmentFile->writeInt((int)($version & 0xFFFFFFFF)); + + // Write segment name counter + $newSegmentFile->writeInt($segmentsFile->readInt()); + + // Get number of segments offset + $numOfSegmentsOffset = $newSegmentFile->tell(); + // Write number of segemnts + $segmentsCount = $segmentsFile->readInt(); + $newSegmentFile->writeInt(0); // Write dummy data (segment counter) + + $segments = array(); + for ($count = 0; $count < $segmentsCount; $count++) { + $segName = $segmentsFile->readString(); + $segSize = $segmentsFile->readInt(); + + if (!in_array($segName, $this->_segmentsToDelete)) { + $newSegmentFile->writeString($segName); + $newSegmentFile->writeInt($segSize); + + $segments[$segName] = $segSize; + } + } + $segmentsFile->close(); + + $segmentsCount = count($segments) + count($this->_newSegments); + + // Remove segments, not listed in $segments (deleted) + // Load segments, not listed in $this->_segmentInfos + foreach ($this->_segmentInfos as $segId => $segInfo) { + if (isset($segments[$segInfo->getName()])) { + // Segment is already included into $this->_segmentInfos + unset($segments[$segInfo->getName()]); + } else { + // remove deleted segment from a list + unset($this->_segmentInfos[$segId]); + } + } + // $segments contains a list of segments to load + // do it later + + foreach ($this->_newSegments as $segName => $segmentInfo) { + $newSegmentFile->writeString($segName); + $newSegmentFile->writeInt($segmentInfo->count()); + + $this->_segmentInfos[] = $segmentInfo; + } + $this->_newSegments = array(); + + $newSegmentFile->seek($numOfSegmentsOffset); + $newSegmentFile->writeInt($segmentsCount); // Update segments count + $newSegmentFile->close(); $this->_directory->renameFile('segments.new', 'segments'); + + + // Segments file update is finished + // Switch back to shared lock mode + $lock->lock(LOCK_SH); + + + $fileList = $this->_directory->fileList(); + foreach ($this->_segmentsToDelete as $nameToDelete) { + foreach (self::$_indexExtensions as $ext) { + if ($this->_directory->fileExists($nameToDelete . $ext)) { + $this->_directory->deleteFile($nameToDelete . $ext); + } + } + + foreach ($fileList as $file) { + if (substr($file, 0, strlen($nameToDelete) + 2) == ($nameToDelete . '.f') && + ctype_digit( substr($file, strlen($nameToDelete) + 2) )) { + $this->_directory->deleteFile($file); + } + } + } + $this->_segmentsToDelete = array(); + + // Load segments, created by other process + foreach ($segments as $segName => $segSize) { + // Load new segments + $this->_segmentInfos[] = new Zend_Search_Lucene_Index_SegmentInfo($segName, + $segSize, + $this->_directory); + } } /** * Commit current changes - * returns array of new segments - * - * @return array */ public function commit() { @@ -255,14 +438,10 @@ class Zend_Search_Lucene_Index_Writer $this->_currentSegment = null; } - if (count($this->_newSegments) != 0) { + if (count($this->_newSegments) != 0 || + count($this->_segmentsToDelete) != 0) { $this->_updateSegments(); } - - $result = $this->_newSegments; - $this->_newSegments = array(); - - return $result; } @@ -279,43 +458,16 @@ class Zend_Search_Lucene_Index_Writer */ } - - /** - * Returns the number of documents currently in this index. - * - * @return integer - */ - public function docCount($readers) - { - /** - * @todo implementation - */ - } - - - /** - * Flushes all changes to an index and closes all associated files. - * - */ - public function close() - { - /** - * @todo implementation - */ - } - - /** * Merges all segments together into a single segment, optimizing * an index for search. + * Input is an array of Zend_Search_Lucene_Index_SegmentInfo objects * - * return void + * @throws Zend_Search_Lucene_Exception */ public function optimize() { - /** - * @todo implementation - */ + $this->_mergeSegments($this->_segmentInfos); } /** @@ -325,7 +477,30 @@ class Zend_Search_Lucene_Index_Writer */ private function _newSegmentName() { - return '_' . base_convert($this->_segmentNameCounter++, 10, 36); + // Do not share file handler to get file updates from other sessions. + $segmentsFile = $this->_directory->getFileObject('segments', false); + + // Get exclusive segments file lock + // We have guarantee, that we will not intersect with _updateSegments() call + // of other process, because it needs exclusive index lock and waits + // until all other searchers won't stop + if (!$segmentsFile->lock(LOCK_EX)) { + throw new Zend_Search_Lucene_Exception('Can\'t obtain exclusive index lock'); + } + + $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version) + $segmentNameCounter = $segmentsFile->readInt(); + + $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version) + $segmentsFile->writeInt($segmentNameCounter + 1); + + // Flash output to guarantee that wrong value will not be loaded between unlock and + // return (which calls $segmentsFile destructor) + $segmentsFile->flush(); + + $segmentsFile->unlock(); + + return '_' . base_convert($segmentNameCounter, 10, 36); } } diff --git a/search/Zend/Search/Lucene/Interface.php b/search/Zend/Search/Lucene/Interface.php new file mode 100644 index 00000000000..58c75b644f3 --- /dev/null +++ b/search/Zend/Search/Lucene/Interface.php @@ -0,0 +1,330 @@ + 10) are best for batch index creation, + * and smaller values (< 10) for indices that are interactively maintained. + * + * Default value is 10 + * + * @return integer + */ + public function getMergeFactor(); + + /** + * Set index mergeFactor option + * + * mergeFactor determines how often segment indices are merged by addDocument(). + * With smaller values, less RAM is used while indexing, + * and searches on unoptimized indices are faster, + * but indexing speed is slower. + * With larger values, more RAM is used during indexing, + * and while searches on unoptimized indices are slower, + * indexing is faster. + * Thus larger values (> 10) are best for batch index creation, + * and smaller values (< 10) for indices that are interactively maintained. + * + * Default value is 10 + * + * @param integer $maxMergeDocs + */ + public function setMergeFactor($mergeFactor); + + /** + * Performs a query against the index and returns an array + * of Zend_Search_Lucene_Search_QueryHit objects. + * Input is a string or Zend_Search_Lucene_Search_Query. + * + * @param mixed $query + * @return array Zend_Search_Lucene_Search_QueryHit + * @throws Zend_Search_Lucene_Exception + */ + public function find($query); + + /** + * Returns a list of all unique field names that exist in this index. + * + * @param boolean $indexed + * @return array + */ + public function getFieldNames($indexed = false); + + /** + * Returns a Zend_Search_Lucene_Document object for the document + * number $id in this index. + * + * @param integer|Zend_Search_Lucene_Search_QueryHit $id + * @return Zend_Search_Lucene_Document + */ + public function getDocument($id); + + /** + * Returns true if index contain documents with specified term. + * + * Is used for query optimization. + * + * @param Zend_Search_Lucene_Index_Term $term + * @return boolean + */ + public function hasTerm(Zend_Search_Lucene_Index_Term $term); + + /** + * Returns IDs of all the documents containing term. + * + * @param Zend_Search_Lucene_Index_Term $term + * @return array + */ + public function termDocs(Zend_Search_Lucene_Index_Term $term); + + /** + * Returns an array of all term freqs. + * Return array structure: array( docId => freq, ...) + * + * @param Zend_Search_Lucene_Index_Term $term + * @return integer + */ + public function termFreqs(Zend_Search_Lucene_Index_Term $term); + + /** + * Returns an array of all term positions in the documents. + * Return array structure: array( docId => array( pos1, pos2, ...), ...) + * + * @param Zend_Search_Lucene_Index_Term $term + * @return array + */ + public function termPositions(Zend_Search_Lucene_Index_Term $term); + + /** + * Returns the number of documents in this index containing the $term. + * + * @param Zend_Search_Lucene_Index_Term $term + * @return integer + */ + public function docFreq(Zend_Search_Lucene_Index_Term $term); + + /** + * Retrive similarity used by index reader + * + * @return Zend_Search_Lucene_Search_Similarity + */ + public function getSimilarity(); + + /** + * Returns a normalization factor for "field, document" pair. + * + * @param integer $id + * @param string $fieldName + * @return float + */ + public function norm($id, $fieldName); + + /** + * Returns true if any documents have been deleted from this index. + * + * @return boolean + */ + public function hasDeletions(); + + /** + * Deletes a document from the index. + * $id is an internal document id + * + * @param integer|Zend_Search_Lucene_Search_QueryHit $id + * @throws Zend_Search_Lucene_Exception + */ + public function delete($id); + + /** + * Adds a document to this index. + * + * @param Zend_Search_Lucene_Document $document + */ + public function addDocument(Zend_Search_Lucene_Document $document); + + /** + * Commit changes resulting from delete() or undeleteAll() operations. + */ + public function commit(); + + /** + * Optimize index. + * + * Merges all segments into one + */ + public function optimize(); + + /** + * Returns an array of all terms in this index. + * + * @return array + */ + public function terms(); + + /** + * Undeletes all documents currently marked as deleted in this index. + */ + public function undeleteAll(); + + + /** + * Add reference to the index object + * + * @internal + */ + public function addReference(); + + /** + * Remove reference from the index object + * + * When reference count becomes zero, index is closed and resources are cleaned up + * + * @internal + */ + public function removeReference(); +} diff --git a/search/Zend/Search/Lucene/PriorityQueue.php b/search/Zend/Search/Lucene/PriorityQueue.php new file mode 100644 index 00000000000..4e844b18f60 --- /dev/null +++ b/search/Zend/Search/Lucene/PriorityQueue.php @@ -0,0 +1,170 @@ +_heap); + $parentId = ($nodeId-1) >> 1; // floor( ($nodeId-1)/2 ) + + while ($nodeId != 0 && $this->_less($element, $this->_heap[$parentId])) { + // Move parent node down + $this->_heap[$nodeId] = $this->_heap[$parentId]; + + // Move pointer to the next level of tree + $nodeId = $parentId; + $parentId = ($nodeId-1) >> 1; // floor( ($nodeId-1)/2 ) + } + + // Put new node into the tree + $this->_heap[$nodeId] = $element; + } + + + /** + * Return least element of the queue + * + * Constant time + * + * @return mixed + */ + public function top() + { + if (count($this->_heap) == 0) { + return null; + } + + return $this->_heap[0]; + } + + + /** + * Removes and return least element of the queue + * + * O(log(N)) time + * + * @return mixed + */ + public function pop() + { + if (count($this->_heap) == 0) { + return null; + } + + $top = $this->_heap[0]; + $lastId = count($this->_heap) - 1; + + /** + * Find appropriate position for last node + */ + $nodeId = 0; // Start from a top + $childId = 1; // First child + + // Choose smaller child + if ($lastId > 2 && $this->_less($this->_heap[2], $this->_heap[1])) { + $childId = 2; + } + + while ($childId < $lastId && + $this->_less($this->_heap[$childId], $this->_heap[$lastId]) + ) { + // Move child node up + $this->_heap[$nodeId] = $this->_heap[$childId]; + + $nodeId = $childId; // Go down + $childId = ($nodeId << 1) + 1; // First child + + // Choose smaller child + if (($childId+1) < $lastId && + $this->_less($this->_heap[$childId+1], $this->_heap[$childId]) + ) { + $childId++; + } + } + + // Move last element to the new position + $this->_heap[$nodeId] = $this->_heap[$lastId]; + unset($this->_heap[$lastId]); + + return $top; + } + + + /** + * Clear queue + */ + public function clear() + { + $this->_heap = array(); + } + + + /** + * Compare elements + * + * Returns true, if $el1 is less than $el2; else otherwise + * + * @param mixed $el1 + * @param mixed $el2 + * @return boolean + */ + abstract protected function _less($el1, $el2); +} + diff --git a/search/Zend/Search/Lucene/Proxy.php b/search/Zend/Search/Lucene/Proxy.php new file mode 100644 index 00000000000..53fb150bd49 --- /dev/null +++ b/search/Zend/Search/Lucene/Proxy.php @@ -0,0 +1,468 @@ +dirroot.'/search/Zend/Search/Lucene/Interface.php'; + + +/** + * Proxy class intended to be used in userland. + * + * It tracks, when index object goes out of scope and forces ndex closing + * + * @category Zend + * @package Zend_Search_Lucene + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Proxy implements Zend_Search_Lucene_Interface +{ + /** + * Index object + * + * @var Zend_Search_Lucene_Interface + */ + private $_index; + + /** + * Object constructor + * + * @param Zend_Search_Lucene_Interface $index + */ + public function __construct(Zend_Search_Lucene_Interface $index) + { + $this->_index = $index; + $this->_index->addReference(); + } + + /** + * Object destructor + */ + public function __destruct() + { + if ($this->_index !== null) { + // This code is invoked if Zend_Search_Lucene_Interface object constructor throws an exception + $this->_index->removeReference(); + } + $this->_index = null; + } + + /** + * Returns the Zend_Search_Lucene_Storage_Directory instance for this index. + * + * @return Zend_Search_Lucene_Storage_Directory + */ + public function getDirectory() + { + return $this->_index->getDirectory(); + } + + /** + * Returns the total number of documents in this index (including deleted documents). + * + * @return integer + */ + public function count() + { + return $this->_index->count(); + } + + /** + * Returns one greater than the largest possible document number. + * This may be used to, e.g., determine how big to allocate a structure which will have + * an element for every document number in an index. + * + * @return integer + */ + public function maxDoc() + { + return $this->_index->maxDoc(); + } + + /** + * Returns the total number of non-deleted documents in this index. + * + * @return integer + */ + public function numDocs() + { + return $this->_index->numDocs(); + } + + /** + * Checks, that document is deleted + * + * @param integer $id + * @return boolean + * @throws Zend_Search_Lucene_Exception Exception is thrown if $id is out of the range + */ + public function isDeleted($id) + { + return $this->_index->isDeleted($id); + } + + /** + * Set default search field. + * + * Null means, that search is performed through all fields by default + * + * Default value is null + * + * @param string $fieldName + */ + public static function setDefaultSearchField($fieldName) + { + Zend_Search_Lucene::setDefaultSearchField($fieldName); + } + + /** + * Get default search field. + * + * Null means, that search is performed through all fields by default + * + * @return string + */ + public static function getDefaultSearchField() + { + return Zend_Search_Lucene::getDefaultSearchField(); + } + + /** + * Retrieve index maxBufferedDocs option + * + * maxBufferedDocs is a minimal number of documents required before + * the buffered in-memory documents are written into a new Segment + * + * Default value is 10 + * + * @return integer + */ + public function getMaxBufferedDocs() + { + return $this->_index->getMaxBufferedDocs(); + } + + /** + * Set index maxBufferedDocs option + * + * maxBufferedDocs is a minimal number of documents required before + * the buffered in-memory documents are written into a new Segment + * + * Default value is 10 + * + * @param integer $maxBufferedDocs + */ + public function setMaxBufferedDocs($maxBufferedDocs) + { + $this->_index->setMaxBufferedDocs($maxBufferedDocs); + } + + + /** + * Retrieve index maxMergeDocs option + * + * maxMergeDocs is a largest number of documents ever merged by addDocument(). + * Small values (e.g., less than 10,000) are best for interactive indexing, + * as this limits the length of pauses while indexing to a few seconds. + * Larger values are best for batched indexing and speedier searches. + * + * Default value is PHP_INT_MAX + * + * @return integer + */ + public function getMaxMergeDocs() + { + return $this->_index->getMaxMergeDocs(); + } + + /** + * Set index maxMergeDocs option + * + * maxMergeDocs is a largest number of documents ever merged by addDocument(). + * Small values (e.g., less than 10,000) are best for interactive indexing, + * as this limits the length of pauses while indexing to a few seconds. + * Larger values are best for batched indexing and speedier searches. + * + * Default value is PHP_INT_MAX + * + * @param integer $maxMergeDocs + */ + public function setMaxMergeDocs($maxMergeDocs) + { + $this->_index->setMaxMergeDocs($maxMergeDocs); + } + + + /** + * Retrieve index mergeFactor option + * + * mergeFactor determines how often segment indices are merged by addDocument(). + * With smaller values, less RAM is used while indexing, + * and searches on unoptimized indices are faster, + * but indexing speed is slower. + * With larger values, more RAM is used during indexing, + * and while searches on unoptimized indices are slower, + * indexing is faster. + * Thus larger values (> 10) are best for batch index creation, + * and smaller values (< 10) for indices that are interactively maintained. + * + * Default value is 10 + * + * @return integer + */ + public function getMergeFactor() + { + return $this->_index->getMergeFactor(); + } + + /** + * Set index mergeFactor option + * + * mergeFactor determines how often segment indices are merged by addDocument(). + * With smaller values, less RAM is used while indexing, + * and searches on unoptimized indices are faster, + * but indexing speed is slower. + * With larger values, more RAM is used during indexing, + * and while searches on unoptimized indices are slower, + * indexing is faster. + * Thus larger values (> 10) are best for batch index creation, + * and smaller values (< 10) for indices that are interactively maintained. + * + * Default value is 10 + * + * @param integer $maxMergeDocs + */ + public function setMergeFactor($mergeFactor) + { + $this->_index->setMergeFactor($mergeFactor); + } + + /** + * Performs a query against the index and returns an array + * of Zend_Search_Lucene_Search_QueryHit objects. + * Input is a string or Zend_Search_Lucene_Search_Query. + * + * @param mixed $query + * @return array Zend_Search_Lucene_Search_QueryHit + * @throws Zend_Search_Lucene_Exception + */ + public function find($query) + { + // actual parameter list + $parameters = func_get_args(); + + // invoke $this->_index->find() method with specified parameters + return call_user_func_array(array(&$this->_index, 'find'), $parameters); + } + + /** + * Returns a list of all unique field names that exist in this index. + * + * @param boolean $indexed + * @return array + */ + public function getFieldNames($indexed = false) + { + return $this->_index->getFieldNames($indexed); + } + + /** + * Returns a Zend_Search_Lucene_Document object for the document + * number $id in this index. + * + * @param integer|Zend_Search_Lucene_Search_QueryHit $id + * @return Zend_Search_Lucene_Document + */ + public function getDocument($id) + { + return $this->_index->getDocument($id); + } + + /** + * Returns true if index contain documents with specified term. + * + * Is used for query optimization. + * + * @param Zend_Search_Lucene_Index_Term $term + * @return boolean + */ + public function hasTerm(Zend_Search_Lucene_Index_Term $term) + { + return $this->_index->hasTerm($term); + } + + /** + * Returns IDs of all the documents containing term. + * + * @param Zend_Search_Lucene_Index_Term $term + * @return array + */ + public function termDocs(Zend_Search_Lucene_Index_Term $term) + { + return $this->_index->termDocs($term); + } + + /** + * Returns an array of all term freqs. + * Return array structure: array( docId => freq, ...) + * + * @param Zend_Search_Lucene_Index_Term $term + * @return integer + */ + public function termFreqs(Zend_Search_Lucene_Index_Term $term) + { + return $this->_index->termFreqs($term); + } + + /** + * Returns an array of all term positions in the documents. + * Return array structure: array( docId => array( pos1, pos2, ...), ...) + * + * @param Zend_Search_Lucene_Index_Term $term + * @return array + */ + public function termPositions(Zend_Search_Lucene_Index_Term $term) + { + return $this->_index->termPositions($term); + } + + /** + * Returns the number of documents in this index containing the $term. + * + * @param Zend_Search_Lucene_Index_Term $term + * @return integer + */ + public function docFreq(Zend_Search_Lucene_Index_Term $term) + { + return $this->_index->docFreq($term); + } + + /** + * Retrive similarity used by index reader + * + * @return Zend_Search_Lucene_Search_Similarity + */ + public function getSimilarity() + { + return $this->_index->getSimilarity(); + } + + /** + * Returns a normalization factor for "field, document" pair. + * + * @param integer $id + * @param string $fieldName + * @return float + */ + public function norm($id, $fieldName) + { + return $this->_index->norm($id, $fieldName); + } + + /** + * Returns true if any documents have been deleted from this index. + * + * @return boolean + */ + public function hasDeletions() + { + return $this->_index->hasDeletions(); + } + + /** + * Deletes a document from the index. + * $id is an internal document id + * + * @param integer|Zend_Search_Lucene_Search_QueryHit $id + * @throws Zend_Search_Lucene_Exception + */ + public function delete($id) + { + return $this->_index->delete($id); + } + + /** + * Adds a document to this index. + * + * @param Zend_Search_Lucene_Document $document + */ + public function addDocument(Zend_Search_Lucene_Document $document) + { + $this->_index->addDocument($document); + } + + /** + * Commit changes resulting from delete() or undeleteAll() operations. + */ + public function commit() + { + $this->_index->commit(); + } + + /** + * Optimize index. + * + * Merges all segments into one + */ + public function optimize() + { + $this->_index->optimize(); + } + + /** + * Returns an array of all terms in this index. + * + * @return array + */ + public function terms() + { + return $this->_index->terms(); + } + + /** + * Undeletes all documents currently marked as deleted in this index. + */ + public function undeleteAll() + { + return $this->_index->undeleteAll(); + } + + /** + * Add reference to the index object + * + * @internal + */ + public function addReference() + { + return $this->_index->addReference(); + } + + /** + * Remove reference from the index object + * + * When reference count becomes zero, index is closed and resources are cleaned up + * + * @internal + */ + public function removeReference() + { + return $this->_index->removeReference(); + } +} diff --git a/search/Zend/Search/Lucene/Search/BooleanExpressionRecognizer.php b/search/Zend/Search/Lucene/Search/BooleanExpressionRecognizer.php new file mode 100644 index 00000000000..1801bc1cf3a --- /dev/null +++ b/search/Zend/Search/Lucene/Search/BooleanExpressionRecognizer.php @@ -0,0 +1,280 @@ +dirroot.'/search/Zend/Search/Lucene/FSM.php'; + +/** Zend_Search_Lucene_Search_QueryToken */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryToken.php'; + +/** Zend_Search_Lucene_Search_QueryParser */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParser.php'; + + +/** Zend_Search_Lucene_Exception */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Search + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Search_BooleanExpressionRecognizer extends Zend_Search_Lucene_FSM +{ + /** State Machine states */ + const ST_START = 0; + const ST_LITERAL = 1; + const ST_NOT_OPERATOR = 2; + const ST_AND_OPERATOR = 3; + const ST_OR_OPERATOR = 4; + + /** Input symbols */ + const IN_LITERAL = 0; + const IN_NOT_OPERATOR = 1; + const IN_AND_OPERATOR = 2; + const IN_OR_OPERATOR = 3; + + + /** + * NOT operator signal + * + * @var boolean + */ + private $_negativeLiteral = false; + + /** + * Current literal + * + * @var mixed + */ + private $_literal; + + + /** + * Set of boolean query conjunctions + * + * Each conjunction is an array of conjunction elements + * Each conjunction element is presented with two-elements array: + * array(, ) + * + * So, it has a structure: + * array( array( array(, ), // first literal of first conjuction + * array(, ), // second literal of first conjuction + * ... + * array(, ) + * ), // end of first conjuction + * array( array(, ), // first literal of second conjuction + * array(, ), // second literal of second conjuction + * ... + * array(, ) + * ), // end of second conjuction + * ... + * ) // end of structure + * + * @var array + */ + private $_conjunctions = array(); + + /** + * Current conjuction + * + * @var array + */ + private $_currentConjunction = array(); + + + /** + * Object constructor + */ + public function __construct() + { + parent::__construct( array(self::ST_START, + self::ST_LITERAL, + self::ST_NOT_OPERATOR, + self::ST_AND_OPERATOR, + self::ST_OR_OPERATOR), + array(self::IN_LITERAL, + self::IN_NOT_OPERATOR, + self::IN_AND_OPERATOR, + self::IN_OR_OPERATOR)); + + $emptyOperatorAction = new Zend_Search_Lucene_FSMAction($this, 'emptyOperatorAction'); + $emptyNotOperatorAction = new Zend_Search_Lucene_FSMAction($this, 'emptyNotOperatorAction'); + + $this->addRules(array( array(self::ST_START, self::IN_LITERAL, self::ST_LITERAL), + array(self::ST_START, self::IN_NOT_OPERATOR, self::ST_NOT_OPERATOR), + + array(self::ST_LITERAL, self::IN_AND_OPERATOR, self::ST_AND_OPERATOR), + array(self::ST_LITERAL, self::IN_OR_OPERATOR, self::ST_OR_OPERATOR), + array(self::ST_LITERAL, self::IN_LITERAL, self::ST_LITERAL, $emptyOperatorAction), + array(self::ST_LITERAL, self::IN_NOT_OPERATOR, self::ST_NOT_OPERATOR, $emptyNotOperatorAction), + + array(self::ST_NOT_OPERATOR, self::IN_LITERAL, self::ST_LITERAL), + + array(self::ST_AND_OPERATOR, self::IN_LITERAL, self::ST_LITERAL), + array(self::ST_AND_OPERATOR, self::IN_NOT_OPERATOR, self::ST_NOT_OPERATOR), + + array(self::ST_OR_OPERATOR, self::IN_LITERAL, self::ST_LITERAL), + array(self::ST_OR_OPERATOR, self::IN_NOT_OPERATOR, self::ST_NOT_OPERATOR), + )); + + $notOperatorAction = new Zend_Search_Lucene_FSMAction($this, 'notOperatorAction'); + $orOperatorAction = new Zend_Search_Lucene_FSMAction($this, 'orOperatorAction'); + $literalAction = new Zend_Search_Lucene_FSMAction($this, 'literalAction'); + + + $this->addEntryAction(self::ST_NOT_OPERATOR, $notOperatorAction); + $this->addEntryAction(self::ST_OR_OPERATOR, $orOperatorAction); + $this->addEntryAction(self::ST_LITERAL, $literalAction); + } + + + /** + * Process next operator. + * + * Operators are defined by class constants: IN_AND_OPERATOR, IN_OR_OPERATOR and IN_NOT_OPERATOR + * + * @param integer $operator + */ + public function processOperator($operator) + { + $this->process($operator); + } + + /** + * Process expression literal. + * + * @param integer $operator + */ + public function processLiteral($literal) + { + $this->_literal = $literal; + + $this->process(self::IN_LITERAL); + } + + /** + * Finish an expression and return result + * + * Result is a set of boolean query conjunctions + * + * Each conjunction is an array of conjunction elements + * Each conjunction element is presented with two-elements array: + * array(, ) + * + * So, it has a structure: + * array( array( array(, ), // first literal of first conjuction + * array(, ), // second literal of first conjuction + * ... + * array(, ) + * ), // end of first conjuction + * array( array(, ), // first literal of second conjuction + * array(, ), // second literal of second conjuction + * ... + * array(, ) + * ), // end of second conjuction + * ... + * ) // end of structure + * + * @return array + * @throws Zend_Search_Lucene_Exception + */ + public function finishExpression() + { + if ($this->getState() != self::ST_LITERAL) { + throw new Zend_Search_Lucene_Exception('Literal expected.'); + } + + $this->_conjunctions[] = $this->_currentConjunction; + + return $this->_conjunctions; + } + + + + /********************************************************************* + * Actions implementation + *********************************************************************/ + + /** + * default (omitted) operator processing + */ + public function emptyOperatorAction() + { + if (Zend_Search_Lucene_Search_QueryParser::getDefaultOperator() == Zend_Search_Lucene_Search_QueryParser::B_AND) { + // Do nothing + } else { + $this->orOperatorAction(); + } + + // Process literal + $this->literalAction(); + } + + /** + * default (omitted) + NOT operator processing + */ + public function emptyNotOperatorAction() + { + if (Zend_Search_Lucene_Search_QueryParser::getDefaultOperator() == Zend_Search_Lucene_Search_QueryParser::B_AND) { + // Do nothing + } else { + $this->orOperatorAction(); + } + + // Process NOT operator + $this->notOperatorAction(); + } + + + /** + * NOT operator processing + */ + public function notOperatorAction() + { + $this->_negativeLiteral = true; + } + + /** + * OR operator processing + * Close current conjunction + */ + public function orOperatorAction() + { + $this->_conjunctions[] = $this->_currentConjunction; + $this->_currentConjunction = array(); + } + + /** + * Literal processing + */ + public function literalAction() + { + // Add literal to the current conjunction + $this->_currentConjunction[] = array($this->_literal, !$this->_negativeLiteral); + + // Switch off negative signal + $this->_negativeLiteral = false; + } +} diff --git a/search/Zend/Search/Lucene/Search/Query.php b/search/Zend/Search/Lucene/Search/Query.php index bf284970a16..e55c22ce013 100644 --- a/search/Zend/Search/Lucene/Search/Query.php +++ b/search/Zend/Search/Lucene/Search/Query.php @@ -15,16 +15,19 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ +/** Zend_Search_Lucene_Document_Html */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Document/Html.php'; + /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_Search_Query @@ -35,14 +38,31 @@ abstract class Zend_Search_Lucene_Search_Query * * @var float */ - private $_boost = 1.0; + private $_boost = 1; /** * Query weight * * @var Zend_Search_Lucene_Search_Weight */ - protected $_weight; + protected $_weight = null; + + /** + * Current highlight color + * + * @var integer + */ + private $_currentColorIndex = 0; + + /** + * List of colors for text highlighting + * + * @var array + */ + private $_highlightColors = array('#66ffff', '#ff66ff', '#ffff66', + '#ff8888', '#88ff88', '#8888ff', + '#88dddd', '#dd88dd', '#dddd88', + '#aaddff', '#aaffdd', '#ddaaff', '#ddffaa', '#ffaadd', '#ffddaa'); /** @@ -71,30 +91,133 @@ abstract class Zend_Search_Lucene_Search_Query * Score specified document * * @param integer $docId - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader * @return float */ - abstract public function score($docId, $reader); + abstract public function score($docId, Zend_Search_Lucene_Interface $reader); + + /** + * Get document ids likely matching the query + * + * It's an array with document ids as keys (performance considerations) + * + * @return array + */ + abstract public function matchedDocs(); + + /** + * Execute query in context of index reader + * It also initializes necessary internal structures + * + * Query specific implementation + * + * @param Zend_Search_Lucene_Interface $reader + */ + abstract public function execute(Zend_Search_Lucene_Interface $reader); /** * Constructs an appropriate Weight implementation for this query. * - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader * @return Zend_Search_Lucene_Search_Weight */ - abstract protected function _createWeight($reader); + abstract public function createWeight(Zend_Search_Lucene_Interface $reader); /** - * Constructs an initializes a Weight for a query. + * Constructs an initializes a Weight for a _top-level_query_. * - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader */ - protected function _initWeight($reader) + protected function _initWeight(Zend_Search_Lucene_Interface $reader) { - $this->_weight = $this->_createWeight($reader); + // Check, that it's a top-level query and query weight is not initialized yet. + if ($this->_weight !== null) { + return $this->_weight; + } + + $this->createWeight($reader); $sum = $this->_weight->sumOfSquaredWeights(); $queryNorm = $reader->getSimilarity()->queryNorm($sum); $this->_weight->normalize($queryNorm); } -} \ No newline at end of file + /** + * Re-write query into primitive queries in the context of specified index + * + * @param Zend_Search_Lucene_Interface $index + * @return Zend_Search_Lucene_Search_Query + */ + abstract public function rewrite(Zend_Search_Lucene_Interface $index); + + /** + * Optimize query in the context of specified index + * + * @param Zend_Search_Lucene_Interface $index + * @return Zend_Search_Lucene_Search_Query + */ + abstract public function optimize(Zend_Search_Lucene_Interface $index); + + /** + * Reset query, so it can be reused within other queries or + * with other indeces + */ + public function reset() + { + $this->_weight = null; + } + + + /** + * Print a query + * + * @return string + */ + abstract public function __toString(); + + /** + * Return query terms + * + * @return array + */ + abstract public function getQueryTerms(); + + /** + * Get highlight color and shift to next + * + * @param integer &$colorIndex + * @return string + */ + protected function _getHighlightColor(&$colorIndex) + { + $color = $this->_highlightColors[$colorIndex++]; + + $colorIndex %= count($this->_highlightColors); + + return $color; + } + + /** + * Highlight query terms + * + * @param integer &$colorIndex + * @param Zend_Search_Lucene_Document_Html $doc + */ + abstract public function highlightMatchesDOM(Zend_Search_Lucene_Document_Html $doc, &$colorIndex); + + /** + * Highlight matches in $inputHTML + * + * @param string $inputHTML + * @return string + */ + public function highlightMatches($inputHTML) + { + $doc = Zend_Search_Lucene_Document_Html::loadHTML($inputHTML); + + $colorIndex = 0; + $this->highlightMatchesDOM($doc, $colorIndex); + + return $doc->getHTML(); + } +} + diff --git a/search/Zend/Search/Lucene/Search/Query/Boolean.php b/search/Zend/Search/Lucene/Search/Query/Boolean.php new file mode 100644 index 00000000000..710a41f381f --- /dev/null +++ b/search/Zend/Search/Lucene/Search/Query/Boolean.php @@ -0,0 +1,715 @@ +dirroot.'/search/Zend/Search/Lucene/Search/Query.php'; + +/** Zend_Search_Lucene_Search_Weight_Boolean */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/Boolean.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Search + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Search_Query_Boolean extends Zend_Search_Lucene_Search_Query +{ + + /** + * Subqueries + * Array of Zend_Search_Lucene_Query + * + * @var array + */ + private $_subqueries = array(); + + /** + * Subqueries signs. + * If true then subquery is required. + * If false then subquery is prohibited. + * If null then subquery is neither prohibited, nor required + * + * If array is null then all subqueries are required + * + * @var array + */ + private $_signs = array(); + + /** + * Result vector. + * + * @var array + */ + private $_resVector = null; + + /** + * A score factor based on the fraction of all query subqueries + * that a document contains. + * float for conjunction queries + * array of float for non conjunction queries + * + * @var mixed + */ + private $_coord = null; + + + /** + * Class constructor. Create a new Boolean query object. + * + * if $signs array is omitted then all subqueries are required + * it differs from addSubquery() behavior, but should never be used + * + * @param array $subqueries Array of Zend_Search_Search_Query objects + * @param array $signs Array of signs. Sign is boolean|null. + * @return void + */ + public function __construct($subqueries = null, $signs = null) + { + if (is_array($subqueries)) { + $this->_subqueries = $subqueries; + + $this->_signs = null; + // Check if all subqueries are required + if (is_array($signs)) { + foreach ($signs as $sign ) { + if ($sign !== true) { + $this->_signs = $signs; + break; + } + } + } + } + } + + + /** + * Add a $subquery (Zend_Search_Lucene_Query) to this query. + * + * The sign is specified as: + * TRUE - subquery is required + * FALSE - subquery is prohibited + * NULL - subquery is neither prohibited, nor required + * + * @param Zend_Search_Lucene_Search_Query $subquery + * @param boolean|null $sign + * @return void + */ + public function addSubquery(Zend_Search_Lucene_Search_Query $subquery, $sign=null) { + if ($sign !== true || $this->_signs !== null) { // Skip, if all subqueries are required + if ($this->_signs === null) { // Check, If all previous subqueries are required + foreach ($this->_subqueries as $prevSubquery) { + $this->_signs[] = true; + } + } + $this->_signs[] = $sign; + } + + $this->_subqueries[] = $subquery; + } + + /** + * Re-write queries into primitive queries + * + * @param Zend_Search_Lucene_Interface $index + * @return Zend_Search_Lucene_Search_Query + */ + public function rewrite(Zend_Search_Lucene_Interface $index) + { + $query = new Zend_Search_Lucene_Search_Query_Boolean(); + $query->setBoost($this->getBoost()); + + foreach ($this->_subqueries as $subqueryId => $subquery) { + $query->addSubquery($subquery->rewrite($index), + ($this->_signs === null)? true : $this->_signs[$subqueryId]); + } + + return $query; + } + + /** + * Optimize query in the context of specified index + * + * @param Zend_Search_Lucene_Interface $index + * @return Zend_Search_Lucene_Search_Query + */ + public function optimize(Zend_Search_Lucene_Interface $index) + { + $subqueries = array(); + $signs = array(); + + // Optimize all subqueries + foreach ($this->_subqueries as $id => $subquery) { + $subqueries[] = $subquery->optimize($index); + $signs[] = ($this->_signs === null)? true : $this->_signs[$id]; + } + + // Check for empty subqueries + foreach ($subqueries as $id => $subquery) { + if ($subquery instanceof Zend_Search_Lucene_Search_Query_Empty) { + if ($signs[$id] === true) { + // Matching is required, but is actually empty + return new Zend_Search_Lucene_Search_Query_Empty(); + } else { + // Matching is optional or prohibited, but is empty + // Remove it from subqueries and signs list + unset($subqueries[$id]); + unset($signs[$id]); + } + } + } + + + // Check if all non-empty subqueries are prohibited + $allProhibited = true; + foreach ($signs as $sign) { + if ($sign !== false) { + $allProhibited = false; + break; + } + } + if ($allProhibited) { + return new Zend_Search_Lucene_Search_Query_Empty(); + } + + + // Check, if reduced subqueries list has only one entry + if (count($subqueries) == 1) { + // It's a query with only one required or optional clause + // (it's already checked, that it's not a prohibited clause) + + if ($this->getBoost() == 1) { + return reset($subqueries); + } + + $optimizedQuery = clone reset($subqueries); + $optimizedQuery->setBoost($optimizedQuery->getBoost()*$this->getBoost()); + + return $optimizedQuery; + } + + + // Check, if reduced subqueries list is empty + if (count($subqueries) == 0) { + return new Zend_Search_Lucene_Search_Query_Empty(); + } + + + // Prepare first candidate for optimized query + $optimizedQuery = new Zend_Search_Lucene_Search_Query_Boolean($subqueries, $signs); + $optimizedQuery->setBoost($this->getBoost()); + + + $terms = array(); + $tsigns = array(); + $boostFactors = array(); + + // Try to decompose term and multi-term subqueries + foreach ($subqueries as $id => $subquery) { + if ($subquery instanceof Zend_Search_Lucene_Search_Query_Term) { + $terms[] = $subquery->getTerm(); + $tsigns[] = $signs[$id]; + $boostFactors[] = $subquery->getBoost(); + + // remove subquery from a subqueries list + unset($subqueries[$id]); + unset($signs[$id]); + } else if ($subquery instanceof Zend_Search_Lucene_Search_Query_MultiTerm) { + $subTerms = $subquery->getTerms(); + $subSigns = $subquery->getSigns(); + + if ($signs[$id] === true) { + // It's a required multi-term subquery. + // Something like '... +(+term1 -term2 term3 ...) ...' + + // Multi-term required subquery can be decomposed only if it contains + // required terms and doesn't contain prohibited terms: + // ... +(+term1 term2 ...) ... => ... +term1 term2 ... + // + // Check this + $hasRequired = false; + $hasProhibited = false; + if ($subSigns === null) { + // All subterms are required + $hasRequired = true; + } else { + foreach ($subSigns as $sign) { + if ($sign === true) { + $hasRequired = true; + } else if ($sign === false) { + $hasProhibited = true; + break; + } + } + } + // Continue if subquery has prohibited terms or doesn't have required terms + if ($hasProhibited || !$hasRequired) { + continue; + } + + foreach ($subTerms as $termId => $term) { + $terms[] = $term; + $tsigns[] = ($subSigns === null)? true : $subSigns[$termId]; + $boostFactors[] = $subquery->getBoost(); + } + + // remove subquery from a subqueries list + unset($subqueries[$id]); + unset($signs[$id]); + + } else { // $signs[$id] === null || $signs[$id] === false + // It's an optional or prohibited multi-term subquery. + // Something like '... (+term1 -term2 term3 ...) ...' + // or + // something like '... -(+term1 -term2 term3 ...) ...' + + // Multi-term optional and required subqueries can be decomposed + // only if all terms are optional. + // + // Check if all terms are optional. + $onlyOptional = true; + if ($subSigns === null) { + // All subterms are required + $onlyOptional = false; + } else { + foreach ($subSigns as $sign) { + if ($sign !== null) { + $onlyOptional = false; + break; + } + } + } + + // Continue if non-optional terms are presented in this multi-term subquery + if (!$onlyOptional) { + continue; + } + + foreach ($subTerms as $termId => $term) { + $terms[] = $term; + $tsigns[] = ($signs[$id] === null)? null /* optional */ : + false /* prohibited */; + $boostFactors[] = $subquery->getBoost(); + } + + // remove subquery from a subqueries list + unset($subqueries[$id]); + unset($signs[$id]); + } + } + } + + + // Check, if there are no decomposed subqueries + if (count($terms) == 0 ) { + // return prepared candidate + return $optimizedQuery; + } + + + // Check, if all subqueries have been decomposed and all terms has the same boost factor + if (count($subqueries) == 0 && count(array_unique($boostFactors)) == 1) { + $optimizedQuery = new Zend_Search_Lucene_Search_Query_MultiTerm($terms, $tsigns); + $optimizedQuery->setBoost(reset($boostFactors)*$this->getBoost()); + + return $optimizedQuery; + } + + + // This boolean query can't be transformed to Term/MultiTerm query and still contains + // several subqueries + + // Separate prohibited terms + $prohibitedTerms = array(); + foreach ($terms as $id => $term) { + if ($tsigns[$id] === false) { + $prohibitedTerms[] = $term; + + unset($terms[$id]); + unset($tsigns[$id]); + unset($boostFactors[$id]); + } + } + + if (count($terms) == 1) { + $clause = new Zend_Search_Lucene_Search_Query_Term(reset($terms)); + $clause->setBoost(reset($boostFactors)); + + $subqueries[] = $clause; + $signs[] = reset($tsigns); + + // Clear terms list + $terms = array(); + } else if (count($terms) > 1 && count(array_unique($boostFactors)) == 1) { + $clause = new Zend_Search_Lucene_Search_Query_MultiTerm($terms, $tsigns); + $clause->setBoost(reset($boostFactors)); + + $subqueries[] = $clause; + // Clause sign is 'required' if clause contains required terms. 'Optional' otherwise. + $signs[] = (in_array(true, $tsigns))? true : null; + + // Clear terms list + $terms = array(); + } + + if (count($prohibitedTerms) == 1) { + // (boost factors are not significant for prohibited clauses) + $subqueries[] = new Zend_Search_Lucene_Search_Query_Term(reset($prohibitedTerms)); + $signs[] = false; + + // Clear prohibited terms list + $prohibitedTerms = array(); + } else if (count($prohibitedTerms) > 1) { + // prepare signs array + $prohibitedSigns = array(); + foreach ($prohibitedTerms as $id => $term) { + // all prohibited term are grouped as optional into multi-term query + $prohibitedSigns[$id] = null; + } + + // (boost factors are not significant for prohibited clauses) + $subqueries[] = new Zend_Search_Lucene_Search_Query_MultiTerm($prohibitedTerms, $prohibitedSigns); + // Clause sign is 'prohibited' + $signs[] = false; + + // Clear terms list + $prohibitedTerms = array(); + } + + /** @todo Group terms with the same boost factors together */ + + // Check, that all terms are processed + // Replace candidate for optimized query + if (count($terms) == 0 && count($prohibitedTerms) == 0) { + $optimizedQuery = new Zend_Search_Lucene_Search_Query_Boolean($subqueries, $signs); + $optimizedQuery->setBoost($this->getBoost()); + } + + return $optimizedQuery; + } + + /** + * Returns subqueries + * + * @return array + */ + public function getSubqueries() + { + return $this->_subqueries; + } + + + /** + * Return subqueries signs + * + * @return array + */ + public function getSigns() + { + return $this->_signs; + } + + + /** + * Constructs an appropriate Weight implementation for this query. + * + * @param Zend_Search_Lucene_Interface $reader + * @return Zend_Search_Lucene_Search_Weight + */ + public function createWeight(Zend_Search_Lucene_Interface $reader) + { + $this->_weight = new Zend_Search_Lucene_Search_Weight_Boolean($this, $reader); + return $this->_weight; + } + + + /** + * Calculate result vector for Conjunction query + * (like ' AND AND ') + */ + private function _calculateConjunctionResult() + { + $this->_resVector = null; + + if (count($this->_subqueries) == 0) { + $this->_resVector = array(); + } + + foreach ($this->_subqueries as $subquery) { + if($this->_resVector === null) { + $this->_resVector = $subquery->matchedDocs(); + } else { + $this->_resVector = array_intersect_key($this->_resVector, $subquery->matchedDocs()); + } + + if (count($this->_resVector) == 0) { + // Empty result set, we don't need to check other terms + break; + } + } + + ksort($this->_resVector, SORT_NUMERIC); + } + + + /** + * Calculate result vector for non Conjunction query + * (like ' AND AND NOT OR ') + */ + private function _calculateNonConjunctionResult() + { + $required = null; + $optional = array(); + + foreach ($this->_subqueries as $subqueryId => $subquery) { + $docs = $subquery->matchedDocs(); + + if ($this->_signs[$subqueryId] === true) { + // required + if ($required !== null) { + // array intersection + $required = array_intersect_key($required, $docs); + } else { + $required = $docs; + } + } elseif ($this->_signs[$subqueryId] === false) { + // prohibited + // Do nothing. matchedDocs() may include non-matching id's + } else { + // neither required, nor prohibited + // array union + $optional += $docs; + } + } + + if ($required !== null) { + $this->_resVector = &$required; + } else { + $this->_resVector = &$optional; + } + + ksort($this->_resVector, SORT_NUMERIC); + } + + + /** + * Score calculator for conjunction queries (all subqueries are required) + * + * @param integer $docId + * @param Zend_Search_Lucene_Interface $reader + * @return float + */ + public function _conjunctionScore($docId, Zend_Search_Lucene_Interface $reader) + { + if ($this->_coord === null) { + $this->_coord = $reader->getSimilarity()->coord(count($this->_subqueries), + count($this->_subqueries) ); + } + + $score = 0; + + foreach ($this->_subqueries as $subquery) { + $subscore = $subquery->score($docId, $reader); + + if ($subscore == 0) { + return 0; + } + + $score += $subquery->score($docId, $reader) * $this->_coord; + } + + return $score * $this->_coord * $this->getBoost(); + } + + + /** + * Score calculator for non conjunction queries (not all subqueries are required) + * + * @param integer $docId + * @param Zend_Search_Lucene_Interface $reader + * @return float + */ + public function _nonConjunctionScore($docId, Zend_Search_Lucene_Interface $reader) + { + if ($this->_coord === null) { + $this->_coord = array(); + + $maxCoord = 0; + foreach ($this->_signs as $sign) { + if ($sign !== false /* not prohibited */) { + $maxCoord++; + } + } + + for ($count = 0; $count <= $maxCoord; $count++) { + $this->_coord[$count] = $reader->getSimilarity()->coord($count, $maxCoord); + } + } + + $score = 0; + $matchedSubqueries = 0; + foreach ($this->_subqueries as $subqueryId => $subquery) { + $subscore = $subquery->score($docId, $reader); + + // Prohibited + if ($this->_signs[$subqueryId] === false && $subscore != 0) { + return 0; + } + + // is required, but doen't match + if ($this->_signs[$subqueryId] === true && $subscore == 0) { + return 0; + } + + if ($subscore != 0) { + $matchedSubqueries++; + $score += $subscore; + } + } + + return $score * $this->_coord[$matchedSubqueries] * $this->getBoost(); + } + + /** + * Execute query in context of index reader + * It also initializes necessary internal structures + * + * @param Zend_Search_Lucene_Interface $reader + */ + public function execute(Zend_Search_Lucene_Interface $reader) + { + // Initialize weight if it's not done yet + $this->_initWeight($reader); + + foreach ($this->_subqueries as $subquery) { + $subquery->execute($reader); + } + + if ($this->_signs === null) { + $this->_calculateConjunctionResult(); + } else { + $this->_calculateNonConjunctionResult(); + } + } + + + + /** + * Get document ids likely matching the query + * + * It's an array with document ids as keys (performance considerations) + * + * @return array + */ + public function matchedDocs() + { + return $this->_resVector; + } + + /** + * Score specified document + * + * @param integer $docId + * @param Zend_Search_Lucene_Interface $reader + * @return float + */ + public function score($docId, Zend_Search_Lucene_Interface $reader) + { + if (isset($this->_resVector[$docId])) { + if ($this->_signs === null) { + return $this->_conjunctionScore($docId, $reader); + } else { + return $this->_nonConjunctionScore($docId, $reader); + } + } else { + return 0; + } + } + + /** + * Return query terms + * + * @return array + */ + public function getQueryTerms() + { + $terms = array(); + + foreach ($this->_subqueries as $id => $subquery) { + if ($this->_signs === null || $this->_signs[$id] !== false) { + $terms = array_merge($terms, $subquery->getQueryTerms()); + } + } + + return $terms; + } + + /** + * Highlight query terms + * + * @param integer &$colorIndex + * @param Zend_Search_Lucene_Document_Html $doc + */ + public function highlightMatchesDOM(Zend_Search_Lucene_Document_Html $doc, &$colorIndex) + { + foreach ($this->_subqueries as $id => $subquery) { + if ($this->_signs === null || $this->_signs[$id] !== false) { + $subquery->highlightMatchesDOM($doc, $colorIndex); + } + } + } + + /** + * Print a query + * + * @return string + */ + public function __toString() + { + // It's used only for query visualisation, so we don't care about characters escaping + + $query = ''; + + foreach ($this->_subqueries as $id => $subquery) { + if ($id != 0) { + $query .= ' '; + } + + if ($this->_signs === null || $this->_signs[$id] === true) { + $query .= '+'; + } else if ($this->_signs[$id] === false) { + $query .= '-'; + } + + $query .= '(' . $subquery->__toString() . ')'; + + if ($subquery->getBoost() != 1) { + $query .= '^' . $subquery->getBoost(); + } + } + + return $query; + } +} + diff --git a/search/Zend/Search/Lucene/Search/Query/Empty.php b/search/Zend/Search/Lucene/Search/Query/Empty.php new file mode 100644 index 00000000000..738e33232fd --- /dev/null +++ b/search/Zend/Search/Lucene/Search/Query/Empty.php @@ -0,0 +1,139 @@ +dirroot.'/search/Zend/Search/Lucene/Search/Query.php'; + +/** Zend_Search_Lucene_Search_Weight_Empty */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/Empty.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Search + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Search_Query_Empty extends Zend_Search_Lucene_Search_Query +{ + /** + * Re-write query into primitive queries in the context of specified index + * + * @param Zend_Search_Lucene_Interface $index + * @return Zend_Search_Lucene_Search_Query + */ + public function rewrite(Zend_Search_Lucene_Interface $index) + { + return $this; + } + + /** + * Optimize query in the context of specified index + * + * @param Zend_Search_Lucene_Interface $index + * @return Zend_Search_Lucene_Search_Query + */ + public function optimize(Zend_Search_Lucene_Interface $index) + { + // "Empty" query is a primitive query and don't need to be optimized + return $this; + } + + /** + * Constructs an appropriate Weight implementation for this query. + * + * @param Zend_Search_Lucene_Interface $reader + * @return Zend_Search_Lucene_Search_Weight + */ + public function createWeight(Zend_Search_Lucene_Interface $reader) + { + return new Zend_Search_Lucene_Search_Weight_Empty(); + } + + /** + * Execute query in context of index reader + * It also initializes necessary internal structures + * + * @param Zend_Search_Lucene_Interface $reader + */ + public function execute(Zend_Search_Lucene_Interface $reader) + { + // Do nothing + } + + /** + * Get document ids likely matching the query + * + * It's an array with document ids as keys (performance considerations) + * + * @return array + */ + public function matchedDocs() + { + return array(); + } + + /** + * Score specified document + * + * @param integer $docId + * @param Zend_Search_Lucene_Interface $reader + * @return float + */ + public function score($docId, Zend_Search_Lucene_Interface $reader) + { + return 0; + } + + /** + * Return query terms + * + * @return array + */ + public function getQueryTerms() + { + return array(); + } + + /** + * Highlight query terms + * + * @param integer &$colorIndex + * @param Zend_Search_Lucene_Document_Html $doc + */ + public function highlightMatchesDOM(Zend_Search_Lucene_Document_Html $doc, &$colorIndex) + { + // Do nothing + } + + /** + * Print a query + * + * @return string + */ + public function __toString() + { + return ''; + } +} + diff --git a/search/Zend/Search/Lucene/Search/Query/MultiTerm.php b/search/Zend/Search/Lucene/Search/Query/MultiTerm.php index d3ec761bc68..9258279a003 100644 --- a/search/Zend/Search/Lucene/Search/Query/MultiTerm.php +++ b/search/Zend/Search/Lucene/Search/Query/MultiTerm.php @@ -15,23 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Query */ -require_once 'Zend/Search/Lucene/Search/Query.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query.php'; /** Zend_Search_Lucene_Search_Weight_MultiTerm */ -require_once 'Zend/Search/Lucene/Search/Weight/MultiTerm.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/MultiTerm.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Search_Query @@ -55,27 +55,24 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc * * @var array */ - - private $_signs = array(); + private $_signs; /** * Result vector. - * Bitset or array of document IDs - * (depending from Bitset extension availability). * - * @var mixed + * @var array */ private $_resVector = null; /** * Terms positions vectors. * Array of Arrays: - * term1Id => (docId => array( pos1, pos2, ... ), ...) - * term2Id => (docId => array( pos1, pos2, ... ), ...) + * term1Id => (docId => freq, ...) + * term2Id => (docId => freq, ...) * * @var array */ - private $_termsPositions = array(); + private $_termsFreqs = array(); /** @@ -101,15 +98,15 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc /** * Class constructor. Create a new multi-term query object. * + * if $signs array is omitted then all terms are required + * it differs from addTerm() behavior, but should never be used + * * @param array $terms Array of Zend_Search_Lucene_Index_Term objects * @param array $signs Array of signs. Sign is boolean|null. * @return void */ public function __construct($terms = null, $signs = null) { - /** - * @todo Check contents of $terms and $signs before adding them. - */ if (is_array($terms)) { $this->_terms = $terms; @@ -119,7 +116,7 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc foreach ($signs as $sign ) { if ($sign !== true) { $this->_signs = $signs; - continue; + break; } } } @@ -139,25 +136,122 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc * @param boolean|null $sign * @return void */ - public function addTerm(Zend_Search_Lucene_Index_Term $term, $sign=null) { - $this->_terms[] = $term; - - /** - * @todo This is not good. Sometimes $this->_signs is an array, sometimes - * it is null, even when there are terms. It will be changed so that - * it is always an array. - */ - if ($this->_signs === null) { - if ($sign !== null) { - $this->_signs = array(); - foreach ($this->_terms as $term) { - $this->_signs[] = null; + public function addTerm(Zend_Search_Lucene_Index_Term $term, $sign = null) { + if ($sign !== true || $this->_signs !== null) { // Skip, if all terms are required + if ($this->_signs === null) { // Check, If all previous terms are required + foreach ($this->_terms as $prevTerm) { + $this->_signs[] = true; } - $this->_signs[] = $sign; } - } else { $this->_signs[] = $sign; } + + $this->_terms[] = $term; + } + + + /** + * Re-write query into primitive queries in the context of specified index + * + * @param Zend_Search_Lucene_Interface $index + * @return Zend_Search_Lucene_Search_Query + */ + public function rewrite(Zend_Search_Lucene_Interface $index) + { + if (count($this->_terms) == 0) { + return new Zend_Search_Lucene_Search_Query_Empty(); + } + + // Check, that all fields are qualified + $allQualified = true; + foreach ($this->_terms as $term) { + if ($term->field === null) { + $allQualified = false; + break; + } + } + + if ($allQualified) { + return $this; + } else { + /** transform multiterm query to boolean and apply rewrite() method to subqueries. */ + $query = new Zend_Search_Lucene_Search_Query_Boolean(); + $query->setBoost($this->getBoost()); + + foreach ($this->_terms as $termId => $term) { + $subquery = new Zend_Search_Lucene_Search_Query_Term($term); + + $query->addSubquery($subquery->rewrite($index), + ($this->_signs === null)? true : $this->_signs[$termId]); + } + + return $query; + } + } + + /** + * Optimize query in the context of specified index + * + * @param Zend_Search_Lucene_Interface $index + * @return Zend_Search_Lucene_Search_Query + */ + public function optimize(Zend_Search_Lucene_Interface $index) + { + $terms = $this->_terms; + $signs = $this->_signs; + + foreach ($terms as $id => $term) { + if (!$index->hasTerm($term)) { + if ($signs === null || $signs[$id] === true) { + // Term is required + return new Zend_Search_Lucene_Search_Query_Empty(); + } else { + // Term is optional or prohibited + // Remove it from terms and signs list + unset($terms[$id]); + unset($signs[$id]); + } + } + } + + // Check if all presented terms are prohibited + $allProhibited = true; + if ($signs === null) { + $allProhibited = false; + } else { + foreach ($signs as $sign) { + if ($sign !== false) { + $allProhibited = false; + break; + } + } + } + if ($allProhibited) { + return new Zend_Search_Lucene_Search_Query_Empty(); + } + + /** + * @todo make an optimization for repeated terms + * (they may have different signs) + */ + + if (count($terms) == 1) { + // It's already checked, that it's not a prohibited term + + // It's one term query with one required or optional element + $optimizedQuery = new Zend_Search_Lucene_Search_Query_Term(reset($terms)); + $optimizedQuery->setBoost($this->getBoost()); + + return $optimizedQuery; + } + + if (count($terms) == 0) { + return new Zend_Search_Lucene_Search_Query_Empty(); + } + + $optimizedQuery = new Zend_Search_Lucene_Search_Query_MultiTerm($terms, $signs); + $optimizedQuery->setBoost($this->getBoost()); + return $optimizedQuery; } @@ -198,12 +292,13 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc /** * Constructs an appropriate Weight implementation for this query. * - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader * @return Zend_Search_Lucene_Search_Weight */ - protected function _createWeight($reader) + public function createWeight(Zend_Search_Lucene_Interface $reader) { - return new Zend_Search_Lucene_Search_Weight_MultiTerm($this, $reader); + $this->_weight = new Zend_Search_Lucene_Search_Weight_MultiTerm($this, $reader); + return $this->_weight; } @@ -211,38 +306,32 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc * Calculate result vector for Conjunction query * (like '+something +another') * - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader */ - private function _calculateConjunctionResult($reader) + private function _calculateConjunctionResult(Zend_Search_Lucene_Interface $reader) { - if (extension_loaded('bitset')) { - foreach( $this->_terms as $termId=>$term ) { - if($this->_resVector === null) { - $this->_resVector = bitset_from_array($reader->termDocs($term)); - } else { - $this->_resVector = bitset_intersection( - $this->_resVector, - bitset_from_array($reader->termDocs($term)) ); - } + $this->_resVector = null; - $this->_termsPositions[$termId] = $reader->termPositions($term); - } - } else { - foreach( $this->_terms as $termId=>$term ) { - if($this->_resVector === null) { - $this->_resVector = array_flip($reader->termDocs($term)); - } else { - $termDocs = array_flip($reader->termDocs($term)); - foreach($this->_resVector as $key=>$value) { - if (!isset( $termDocs[$key] )) { - unset( $this->_resVector[$key] ); - } - } - } - - $this->_termsPositions[$termId] = $reader->termPositions($term); - } + if (count($this->_terms) == 0) { + $this->_resVector = array(); } + + foreach( $this->_terms as $termId=>$term ) { + if($this->_resVector === null) { + $this->_resVector = array_flip($reader->termDocs($term)); + } else { + $this->_resVector = array_intersect_key($this->_resVector, array_flip($reader->termDocs($term))); + } + + if (count($this->_resVector) == 0) { + // Empty result set, we don't need to check other terms + break; + } + + $this->_termsFreqs[$termId] = $reader->termFreqs($term); + } + + ksort($this->_resVector, SORT_NUMERIC); } @@ -250,89 +339,49 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc * Calculate result vector for non Conjunction query * (like '+something -another') * - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader */ - private function _calculateNonConjunctionResult($reader) + private function _calculateNonConjunctionResult(Zend_Search_Lucene_Interface $reader) { - if (extension_loaded('bitset')) { - $required = null; - $neither = bitset_empty(); - $prohibited = bitset_empty(); + $required = null; + $optional = array(); + $prohibited = array(); - foreach ($this->_terms as $termId => $term) { - $termDocs = bitset_from_array($reader->termDocs($term)); + foreach ($this->_terms as $termId => $term) { + $termDocs = array_flip($reader->termDocs($term)); - if ($this->_signs[$termId] === true) { - // required - if ($required !== null) { - $required = bitset_intersection($required, $termDocs); - } else { - $required = $termDocs; - } - } elseif ($this->_signs[$termId] === false) { - // prohibited - $prohibited = bitset_union($prohibited, $termDocs); + if ($this->_signs[$termId] === true) { + // required + if ($required !== null) { + // array intersection + $required = array_intersect_key($required, $termDocs); } else { - // neither required, nor prohibited - $neither = bitset_union($neither, $termDocs); + $required = $termDocs; } - - $this->_termsPositions[$termId] = $reader->termPositions($term); + } elseif ($this->_signs[$termId] === false) { + // prohibited + // array union + $prohibited += $termDocs; + } else { + // neither required, nor prohibited + // array union + $optional += $termDocs; } - if ($required === null) { - $required = $neither; - } - $this->_resVector = bitset_intersection( $required, - bitset_invert($prohibited, $reader->count()) ); - } else { - $required = null; - $neither = array(); - $prohibited = array(); - - foreach ($this->_terms as $termId => $term) { - $termDocs = array_flip($reader->termDocs($term)); - - if ($this->_signs[$termId] === true) { - // required - if ($required !== null) { - // substitute for bitset_intersection - foreach ($required as $key => $value) { - if (!isset( $termDocs[$key] )) { - unset($required[$key]); - } - } - } else { - $required = $termDocs; - } - } elseif ($this->_signs[$termId] === false) { - // prohibited - // substitute for bitset_union - foreach ($termDocs as $key => $value) { - $prohibited[$key] = $value; - } - } else { - // neither required, nor prohibited - // substitute for bitset_union - foreach ($termDocs as $key => $value) { - $neither[$key] = $value; - } - } - - $this->_termsPositions[$termId] = $reader->termPositions($term); - } - - if ($required === null) { - $required = $neither; - } - - foreach ($required as $key=>$value) { - if (isset( $prohibited[$key] )) { - unset($required[$key]); - } - } - $this->_resVector = $required; + $this->_termsFreqs[$termId] = $reader->termFreqs($term); } + + if ($required !== null) { + $this->_resVector = (count($prohibited) > 0) ? + array_diff_key($required, $prohibited) : + $required; + } else { + $this->_resVector = (count($prohibited) > 0) ? + array_diff_key($optional, $prohibited) : + $optional; + } + + ksort($this->_resVector, SORT_NUMERIC); } @@ -340,10 +389,10 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc * Score calculator for conjunction queries (all terms are required) * * @param integer $docId - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader * @return float */ - public function _conjunctionScore($docId, $reader) + public function _conjunctionScore($docId, Zend_Search_Lucene_Interface $reader) { if ($this->_coord === null) { $this->_coord = $reader->getSimilarity()->coord(count($this->_terms), @@ -353,12 +402,16 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc $score = 0.0; foreach ($this->_terms as $termId=>$term) { - $score += $reader->getSimilarity()->tf(count($this->_termsPositions[$termId][$docId]) ) * + /** + * We don't need to check that term freq is not 0 + * Score calculation is performed only for matched docs + */ + $score += $reader->getSimilarity()->tf($this->_termsFreqs[$termId][$docId]) * $this->_weights[$termId]->getValue() * $reader->norm($docId, $term->field); } - return $score * $this->_coord; + return $score * $this->_coord * $this->getBoost(); } @@ -366,7 +419,7 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc * Score calculator for non conjunction queries (not all terms are required) * * @param integer $docId - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader * @return float */ public function _nonConjunctionScore($docId, $reader) @@ -390,42 +443,65 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc $matchedTerms = 0; foreach ($this->_terms as $termId=>$term) { // Check if term is - if ($this->_signs[$termId] !== false && // not prohibited - isset($this->_termsPositions[$termId][$docId]) // matched + if ($this->_signs[$termId] !== false && // not prohibited + isset($this->_termsFreqs[$termId][$docId]) // matched ) { $matchedTerms++; + + /** + * We don't need to check that term freq is not 0 + * Score calculation is performed only for matched docs + */ $score += - $reader->getSimilarity()->tf(count($this->_termsPositions[$termId][$docId]) ) * + $reader->getSimilarity()->tf($this->_termsFreqs[$termId][$docId]) * $this->_weights[$termId]->getValue() * $reader->norm($docId, $term->field); } } - return $score * $this->_coord[$matchedTerms]; + return $score * $this->_coord[$matchedTerms] * $this->getBoost(); + } + + /** + * Execute query in context of index reader + * It also initializes necessary internal structures + * + * @param Zend_Search_Lucene_Interface $reader + */ + public function execute(Zend_Search_Lucene_Interface $reader) + { + if ($this->_signs === null) { + $this->_calculateConjunctionResult($reader); + } else { + $this->_calculateNonConjunctionResult($reader); + } + + // Initialize weight if it's not done yet + $this->_initWeight($reader); + } + + /** + * Get document ids likely matching the query + * + * It's an array with document ids as keys (performance considerations) + * + * @return array + */ + public function matchedDocs() + { + return $this->_resVector; } /** * Score specified document * * @param integer $docId - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader * @return float */ - public function score($docId, $reader) + public function score($docId, Zend_Search_Lucene_Interface $reader) { - if($this->_resVector === null) { - if ($this->_signs === null) { - $this->_calculateConjunctionResult($reader); - } else { - $this->_calculateNonConjunctionResult($reader); - } - - $this->_initWeight($reader); - } - - if ( (extension_loaded('bitset')) ? - bitset_in($this->_resVector, $docId) : - isset($this->_resVector[$docId]) ) { + if (isset($this->_resVector[$docId])) { if ($this->_signs === null) { return $this->_conjunctionScore($docId, $reader); } else { @@ -435,5 +511,87 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc return 0; } } + + /** + * Return query terms + * + * @return array + */ + public function getQueryTerms() + { + if ($this->_signs === null) { + return $this->_terms; + } + + $terms = array(); + + foreach ($this->_signs as $id => $sign) { + if ($sign !== false) { + $terms[] = $this->_terms[$id]; + } + } + + return $terms; + } + + /** + * Highlight query terms + * + * @param integer &$colorIndex + * @param Zend_Search_Lucene_Document_Html $doc + */ + public function highlightMatchesDOM(Zend_Search_Lucene_Document_Html $doc, &$colorIndex) + { + $words = array(); + + if ($this->_signs === null) { + foreach ($this->_terms as $term) { + $words[] = $term->text; + } + } else { + foreach ($this->_signs as $id => $sign) { + if ($sign !== false) { + $words[] = $this->_terms[$id]->text; + } + } + } + + $doc->highlight($words, $this->_getHighlightColor($colorIndex)); + } + + /** + * Print a query + * + * @return string + */ + public function __toString() + { + // It's used only for query visualisation, so we don't care about characters escaping + + $query = ''; + + foreach ($this->_terms as $id => $term) { + if ($id != 0) { + $query .= ' '; + } + + if ($this->_signs === null || $this->_signs[$id] === true) { + $query .= '+'; + } else if ($this->_signs[$id] === false) { + $query .= '-'; + } + + if ($term->field !== null) { + $query .= $term->field . ':'; + } + $query .= $term->text; + } + + if ($this->getBoost() != 1) { + $query = '(' . $query . ')^' . $this->getBoost(); + } + + return $query; + } } diff --git a/search/Zend/Search/Lucene/Search/Query/Phrase.php b/search/Zend/Search/Lucene/Search/Query/Phrase.php index b1d40b4bea4..3e7b782792f 100644 --- a/search/Zend/Search/Lucene/Search/Query/Phrase.php +++ b/search/Zend/Search/Lucene/Search/Query/Phrase.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -23,12 +23,12 @@ /** * Zend_Search_Lucene_Search_Query */ -require_once 'Zend/Search/Lucene/Search/Query.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query.php'; /** * Zend_Search_Lucene_Search_Weight_MultiTerm */ -require_once 'Zend/Search/Lucene/Search/Weight/Phrase.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/Phrase.php'; /** @@ -37,7 +37,7 @@ require_once 'Zend/Search/Lucene/Search/Weight/Phrase.php'; * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Query_Phrase extends Zend_Search_Lucene_Search_Query @@ -73,16 +73,14 @@ class Zend_Search_Lucene_Search_Query_Phrase extends Zend_Search_Lucene_Search_Q * * The slop is zero by default, requiring exact matches. * - * @var unknown_type + * @var integer */ private $_slop; /** * Result vector. - * Bitset or array of document IDs - * (depending from Bitset extension availability). * - * @var mixed + * @var array */ private $_resVector = null; @@ -183,6 +181,70 @@ class Zend_Search_Lucene_Search_Query_Phrase extends Zend_Search_Lucene_Search_Q } + /** + * Re-write query into primitive queries in the context of specified index + * + * @param Zend_Search_Lucene_Interface $index + * @return Zend_Search_Lucene_Search_Query + */ + public function rewrite(Zend_Search_Lucene_Interface $index) + { + if (count($this->_terms) == 0) { + return new Zend_Search_Lucene_Search_Query_Empty(); + } else if ($this->_terms[0]->field !== null) { + return $this; + } else { + $query = new Zend_Search_Lucene_Search_Query_Boolean(); + $query->setBoost($this->getBoost()); + + foreach ($index->getFieldNames(true) as $fieldName) { + $subquery = new Zend_Search_Lucene_Search_Query_Phrase(); + $subquery->setSlop($this->getSlop()); + + foreach ($this->_terms as $termId => $term) { + $qualifiedTerm = new Zend_Search_Lucene_Index_Term($term->text, $fieldName); + + $subquery->addTerm($qualifiedTerm, $this->_offsets[$termId]); + } + + $query->addSubquery($subquery); + } + + return $query; + } + } + + /** + * Optimize query in the context of specified index + * + * @param Zend_Search_Lucene_Interface $index + * @return Zend_Search_Lucene_Search_Query + */ + public function optimize(Zend_Search_Lucene_Interface $index) + { + // Check, that index contains all phrase terms + foreach ($this->_terms as $term) { + if (!$index->hasTerm($term)) { + return new Zend_Search_Lucene_Search_Query_Empty(); + } + } + + if (count($this->_terms) == 1) { + // It's one term query + $optimizedQuery = new Zend_Search_Lucene_Search_Query_Term(reset($this->_terms)); + $optimizedQuery->setBoost($this->getBoost()); + + return $optimizedQuery; + } + + if (count($this->_terms) == 0) { + return new Zend_Search_Lucene_Search_Query_Empty(); + } + + + return $this; + } + /** * Returns query term * @@ -209,50 +271,13 @@ class Zend_Search_Lucene_Search_Query_Phrase extends Zend_Search_Lucene_Search_Q /** * Constructs an appropriate Weight implementation for this query. * - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader * @return Zend_Search_Lucene_Search_Weight */ - protected function _createWeight($reader) + public function createWeight(Zend_Search_Lucene_Interface $reader) { - return new Zend_Search_Lucene_Search_Weight_Phrase($this, $reader); - } - - - /** - * Calculate result vector - * - * @param Zend_Search_Lucene $reader - */ - private function _calculateResult($reader) - { - if (extension_loaded('bitset')) { - foreach( $this->_terms as $termId=>$term ) { - if($this->_resVector === null) { - $this->_resVector = bitset_from_array($reader->termDocs($term)); - } else { - $this->_resVector = bitset_intersection( - $this->_resVector, - bitset_from_array($reader->termDocs($term)) ); - } - - $this->_termsPositions[$termId] = $reader->termPositions($term); - } - } else { - foreach( $this->_terms as $termId=>$term ) { - if($this->_resVector === null) { - $this->_resVector = array_flip($reader->termDocs($term)); - } else { - $termDocs = array_flip($reader->termDocs($term)); - foreach($this->_resVector as $key=>$value) { - if (!isset( $termDocs[$key] )) { - unset( $this->_resVector[$key] ); - } - } - } - - $this->_termsPositions[$termId] = $reader->termPositions($term); - } - } + $this->_weight = new Zend_Search_Lucene_Search_Weight_Phrase($this, $reader); + return $this->_weight; } @@ -305,10 +330,10 @@ class Zend_Search_Lucene_Search_Query_Phrase extends Zend_Search_Lucene_Search_Q * Score calculator for sloppy phrase queries (terms sequence is fixed) * * @param integer $docId - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader * @return float */ - public function _sloppyPhraseFreq($docId, Zend_Search_Lucene $reader) + public function _sloppyPhraseFreq($docId, Zend_Search_Lucene_Interface $reader) { $freq = 0; @@ -377,50 +402,141 @@ class Zend_Search_Lucene_Search_Query_Phrase extends Zend_Search_Lucene_Search_Q return $freq; } + /** + * Execute query in context of index reader + * It also initializes necessary internal structures + * + * @param Zend_Search_Lucene_Interface $reader + */ + public function execute(Zend_Search_Lucene_Interface $reader) + { + $this->_resVector = null; + + if (count($this->_terms) == 0) { + $this->_resVector = array(); + } + + foreach( $this->_terms as $termId=>$term ) { + if($this->_resVector === null) { + $this->_resVector = array_flip($reader->termDocs($term)); + } else { + $this->_resVector = array_intersect_key($this->_resVector, array_flip($reader->termDocs($term))); + } + + if (count($this->_resVector) == 0) { + // Empty result set, we don't need to check other terms + break; + } + + $this->_termsPositions[$termId] = $reader->termPositions($term); + } + + ksort($this->_resVector, SORT_NUMERIC); + + // Initialize weight if it's not done yet + $this->_initWeight($reader); + } + + /** + * Get document ids likely matching the query + * + * It's an array with document ids as keys (performance considerations) + * + * @return array + */ + public function matchedDocs() + { + return $this->_resVector; + } /** * Score specified document * * @param integer $docId - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader * @return float */ - public function score($docId, $reader) + public function score($docId, Zend_Search_Lucene_Interface $reader) { - // optimize zero-term case - if (count($this->_terms) == 0) { - return 0; - } - - if($this->_resVector === null) { - $this->_calculateResult($reader); - $this->_initWeight($reader); - } - - if ( (extension_loaded('bitset')) ? - bitset_in($this->_resVector, $docId) : - isset($this->_resVector[$docId]) ) { + if (isset($this->_resVector[$docId])) { if ($this->_slop == 0) { $freq = $this->_exactPhraseFreq($docId); } else { $freq = $this->_sloppyPhraseFreq($docId, $reader); } -/* - return $reader->getSimilarity()->tf($freq) * - $this->_weight->getValue() * - $reader->norm($docId, reset($this->_terms)->field); -*/ if ($freq != 0) { $tf = $reader->getSimilarity()->tf($freq); $weight = $this->_weight->getValue(); $norm = $reader->norm($docId, reset($this->_terms)->field); - return $tf*$weight*$norm; + return $tf * $weight * $norm * $this->getBoost(); } + + // Included in result, but culculated freq is zero + return 0; } else { return 0; } } + + /** + * Return query terms + * + * @return array + */ + public function getQueryTerms() + { + return $this->_terms; + } + + /** + * Highlight query terms + * + * @param integer &$colorIndex + * @param Zend_Search_Lucene_Document_Html $doc + */ + public function highlightMatchesDOM(Zend_Search_Lucene_Document_Html $doc, &$colorIndex) + { + $words = array(); + foreach ($this->_terms as $term) { + $words[] = $term->text; + } + + $doc->highlight($words, $this->_getHighlightColor($colorIndex)); + } + + /** + * Print a query + * + * @return string + */ + public function __toString() + { + // It's used only for query visualisation, so we don't care about characters escaping + + $query = ''; + + if (isset($this->_terms[0]) && $this->_terms[0]->field !== null) { + $query .= $this->_terms[0]->field . ':'; + } + + $query .= '"'; + + foreach ($this->_terms as $id => $term) { + if ($id != 0) { + $query .= ' '; + } + $query .= $term->text; + } + + $query .= '"'; + + if ($this->_slop != 0) { + $query .= '~' . $this->_slop; + } + + return $query; + } } diff --git a/search/Zend/Search/Lucene/Search/Query/Term.php b/search/Zend/Search/Lucene/Search/Query/Term.php index b0baf0f5aca..0240104e325 100644 --- a/search/Zend/Search/Lucene/Search/Query/Term.php +++ b/search/Zend/Search/Lucene/Search/Query/Term.php @@ -15,23 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Query */ -require_once 'Zend/Search/Lucene/Search/Query.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query.php'; /** Zend_Search_Lucene_Search_Weight_Term */ -require_once 'Zend/Search/Lucene/Search/Weight/Term.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/Term.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Query_Term extends Zend_Search_Lucene_Search_Query @@ -43,31 +43,20 @@ class Zend_Search_Lucene_Search_Query_Term extends Zend_Search_Lucene_Search_Que */ private $_term; - /** - * Term sign. - * If true then term is required - * If false then term is prohibited. - * - * @var bool - */ - private $_sign; - /** * Documents vector. - * Bitset or array of document IDs - * (depending from Bitset extension availability). * - * @var mixed + * @var array */ private $_docVector = null; /** - * Term positions vector. - * Array: docId => array( pos1, pos2, ... ) + * Term freqs vector. + * array(docId => freq, ...) * * @var array */ - private $_termPositions; + private $_termFreqs; /** @@ -76,53 +65,160 @@ class Zend_Search_Lucene_Search_Query_Term extends Zend_Search_Lucene_Search_Que * @param Zend_Search_Lucene_Index_Term $term * @param boolean $sign */ - public function __construct( $term, $sign = true ) + public function __construct($term) { $this->_term = $term; - $this->_sign = $sign; + } + + /** + * Re-write query into primitive queries in the context of specified index + * + * @param Zend_Search_Lucene_Interface $index + * @return Zend_Search_Lucene_Search_Query + */ + public function rewrite(Zend_Search_Lucene_Interface $index) + { + if ($this->_term->field != null) { + return $this; + } else { + $query = new Zend_Search_Lucene_Search_Query_MultiTerm(); + $query->setBoost($this->getBoost()); + + foreach ($index->getFieldNames(true) as $fieldName) { + $term = new Zend_Search_Lucene_Index_Term($this->_term->text, $fieldName); + + $query->addTerm($term); + } + + return $query->rewrite($index); + } + } + + /** + * Optimize query in the context of specified index + * + * @param Zend_Search_Lucene_Interface $index + * @return Zend_Search_Lucene_Search_Query + */ + public function optimize(Zend_Search_Lucene_Interface $index) + { + // Check, that index contains specified term + if (!$index->hasTerm($this->_term)) { + return new Zend_Search_Lucene_Search_Query_Empty(); + } + + return $this; } /** * Constructs an appropriate Weight implementation for this query. * - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader * @return Zend_Search_Lucene_Search_Weight */ - protected function _createWeight($reader) + public function createWeight(Zend_Search_Lucene_Interface $reader) { - return new Zend_Search_Lucene_Search_Weight_Term($this->_term, $this, $reader); + $this->_weight = new Zend_Search_Lucene_Search_Weight_Term($this->_term, $this, $reader); + return $this->_weight; + } + + /** + * Execute query in context of index reader + * It also initializes necessary internal structures + * + * @param Zend_Search_Lucene_Interface $reader + */ + public function execute(Zend_Search_Lucene_Interface $reader) + { + $this->_docVector = array_flip($reader->termDocs($this->_term)); + $this->_termFreqs = $reader->termFreqs($this->_term); + + // Initialize weight if it's not done yet + $this->_initWeight($reader); + } + + /** + * Get document ids likely matching the query + * + * It's an array with document ids as keys (performance considerations) + * + * @return array + */ + public function matchedDocs() + { + return $this->_docVector; } /** * Score specified document * * @param integer $docId - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader * @return float */ - public function score( $docId, $reader ) + public function score($docId, Zend_Search_Lucene_Interface $reader) { - if($this->_docVector===null) { - if (extension_loaded('bitset')) { - $this->_docVector = bitset_from_array( $reader->termDocs($this->_term) ); - } else { - $this->_docVector = array_flip($reader->termDocs($this->_term)); - } - - $this->_termPositions = $reader->termPositions($this->_term); - $this->_initWeight($reader); - } - - $match = extension_loaded('bitset') ? bitset_in($this->_docVector, $docId) : - isset($this->_docVector[$docId]); - if ($this->_sign && $match) { - return $reader->getSimilarity()->tf(count($this->_termPositions[$docId]) ) * + if (isset($this->_docVector[$docId])) { + return $reader->getSimilarity()->tf($this->_termFreqs[$docId]) * $this->_weight->getValue() * - $reader->norm($docId, $this->_term->field); + $reader->norm($docId, $this->_term->field) * + $this->getBoost(); } else { return 0; } } + + /** + * Return query terms + * + * @return array + */ + public function getQueryTerms() + { + return array($this->_term); + } + + /** + * Return query term + * + * @return Zend_Search_Lucene_Index_Term + */ + public function getTerm() + { + return $this->_term; + } + + /** + * Returns query term + * + * @return array + */ + public function getTerms() + { + return $this->_terms; + } + + /** + * Highlight query terms + * + * @param integer &$colorIndex + * @param Zend_Search_Lucene_Document_Html $doc + */ + public function highlightMatchesDOM(Zend_Search_Lucene_Document_Html $doc, &$colorIndex) + { + $doc->highlight($this->_term->text, $this->_getHighlightColor($colorIndex)); + } + + /** + * Print a query + * + * @return string + */ + public function __toString() + { + // It's used only for query visualisation, so we don't care about characters escaping + return (($this->_term->field === null)? '':$this->_term->field . ':') . $this->_term->text; + } } diff --git a/search/Zend/Search/Lucene/Search/QueryEntry.php b/search/Zend/Search/Lucene/Search/QueryEntry.php new file mode 100644 index 00000000000..53777c8570f --- /dev/null +++ b/search/Zend/Search/Lucene/Search/QueryEntry.php @@ -0,0 +1,87 @@ +dirroot.'/search/Zend/Search/Lucene/Index/Term.php'; + +/** Zend_Search_Lucene_Exception */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; + +/** Zend_Search_Lucene_Search_QueryEntry_Term */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry/Term.php'; + +/** Zend_Search_Lucene_Search_QueryEntry_Phrase */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry/Phrase.php'; + +/** Zend_Search_Lucene_Search_QueryEntry_Subquery */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry/Subquery.php'; + + +/** Zend_Search_Lucene_Search_QueryParserException */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Search + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +abstract class Zend_Search_Lucene_Search_QueryEntry +{ + /** + * Query entry boost factor + * + * @var float + */ + protected $_boost = 1.0; + + + /** + * Process modifier ('~') + * + * @param mixed $parameter + */ + abstract public function processFuzzyProximityModifier($parameter = null); + + + /** + * Transform entry to a subquery + * + * @param string $encoding + * @return Zend_Search_Lucene_Search_Query + */ + abstract public function getQuery($encoding); + + /** + * Boost query entry + * + * @param float $boostFactor + */ + public function boost($boostFactor) + { + $this->_boost *= $boostFactor; + } + + +} diff --git a/search/Zend/Search/Lucene/Search/QueryEntry/Phrase.php b/search/Zend/Search/Lucene/Search/QueryEntry/Phrase.php new file mode 100644 index 00000000000..e90a58b4977 --- /dev/null +++ b/search/Zend/Search/Lucene/Search/QueryEntry/Phrase.php @@ -0,0 +1,147 @@ +dirroot.'/search/Zend/Search/Lucene/Index/Term.php'; + +/** Zend_Search_Lucene_Exception */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; + +/** Zend_Search_Lucene_Search_QueryEntry */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry.php'; + +/** Zend_Search_Lucene_Search_QueryParserException */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php'; + +/** Zend_Search_Lucene_Analysis_Analyzer */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php'; + + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Search + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Search_QueryEntry_Phrase extends Zend_Search_Lucene_Search_QueryEntry +{ + /** + * Phrase value + * + * @var string + */ + private $_phrase; + + /** + * Field + * + * @var string|null + */ + private $_field; + + + /** + * Proximity phrase query + * + * @var boolean + */ + private $_proximityQuery = false; + + /** + * Words distance, used for proximiti queries + * + * @var integer + */ + private $_wordsDistance = 0; + + + /** + * Object constractor + * + * @param string $phrase + * @param string $field + */ + public function __construct($phrase, $field) + { + $this->_phrase = $phrase; + $this->_field = $field; + } + + /** + * Process modifier ('~') + * + * @param mixed $parameter + */ + public function processFuzzyProximityModifier($parameter = null) + { + $this->_proximityQuery = true; + + if ($parameter !== null) { + $this->_wordsDistance = $parameter; + } + } + + /** + * Transform entry to a subquery + * + * @param string $encoding + * @return Zend_Search_Lucene_Search_Query + * @throws Zend_Search_Lucene_Search_QueryParserException + */ + public function getQuery($encoding) + { + if (strpos($this->_phrase, '?') !== false || strpos($this->_phrase, '*') !== false) { + throw new Zend_Search_Lucene_Search_QueryParserException('Wildcards are only allowed in a single terms.'); + } + + $tokens = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($this->_phrase, $encoding); + + if (count($tokens) == 0) { + return new Zend_Search_Lucene_Search_Query_Empty(); + } + + if (count($tokens) == 1) { + $term = new Zend_Search_Lucene_Index_Term($tokens[0]->getTermText(), $this->_field); + $query = new Zend_Search_Lucene_Search_Query_Term($term); + $query->setBoost($this->_boost); + + return $query; + } + + //It's not empty or one term query + $query = new Zend_Search_Lucene_Search_Query_Phrase(); + foreach ($tokens as $token) { + $term = new Zend_Search_Lucene_Index_Term($token->getTermText(), $this->_field); + $query->addTerm($term); + } + + if ($this->_proximityQuery) { + $query->setSlop($this->_wordsDistance); + } + + $query->setBoost($this->_boost); + + return $query; + } +} diff --git a/search/Zend/Search/Lucene/Search/QueryEntry/Subquery.php b/search/Zend/Search/Lucene/Search/QueryEntry/Subquery.php new file mode 100644 index 00000000000..8d6e64fc63d --- /dev/null +++ b/search/Zend/Search/Lucene/Search/QueryEntry/Subquery.php @@ -0,0 +1,86 @@ +dirroot.'/search/Zend/Search/Lucene/Index/Term.php'; + +/** Zend_Search_Lucene_Exception */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; + +/** Zend_Search_Lucene_Search_QueryEntry */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry.php'; + +/** Zend_Search_Lucene_Search_QueryParserException */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Search + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Search_QueryEntry_Subquery extends Zend_Search_Lucene_Search_QueryEntry +{ + /** + * Query + * + * @var Zend_Search_Lucene_Search_Query + */ + private $_query; + + /** + * Object constractor + * + * @param Zend_Search_Lucene_Search_Query $query + */ + public function __construct(Zend_Search_Lucene_Search_Query $query) + { + $this->_query = $query; + } + + /** + * Process modifier ('~') + * + * @param mixed $parameter + * @throws Zend_Search_Lucene_Search_QueryParserException + */ + public function processFuzzyProximityModifier($parameter = null) + { + throw new Zend_Search_Lucene_Search_QueryParserException('\'~\' sign must follow term or phrase'); + } + + + /** + * Transform entry to a subquery + * + * @param string $encoding + * @return Zend_Search_Lucene_Search_Query + */ + public function getQuery($encoding) + { + $this->_query->setBoost($this->_boost); + + return $this->_query; + } +} diff --git a/search/Zend/Search/Lucene/Search/QueryEntry/Term.php b/search/Zend/Search/Lucene/Search/QueryEntry/Term.php new file mode 100644 index 00000000000..f9b3d9746eb --- /dev/null +++ b/search/Zend/Search/Lucene/Search/QueryEntry/Term.php @@ -0,0 +1,154 @@ +dirroot.'/search/Zend/Search/Lucene/Index/Term.php'; + +/** Zend_Search_Lucene_Exception */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; + +/** Zend_Search_Lucene_Search_QueryEntry */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry.php'; + +/** Zend_Search_Lucene_Search_QueryParserException */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php'; + +/** Zend_Search_Lucene_Analysis_Analyzer */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php'; + + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Search + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Search_QueryEntry_Term extends Zend_Search_Lucene_Search_QueryEntry +{ + /** + * Term value + * + * @var string + */ + private $_term; + + /** + * Field + * + * @var string|null + */ + private $_field; + + + /** + * Fuzzy search query + * + * @var boolean + */ + private $_fuzzyQuery = false; + + /** + * Similarity + * + * @var float + */ + private $_similarity = 1.; + + + /** + * Object constractor + * + * @param string $term + * @param string $field + */ + public function __construct($term, $field) + { + $this->_term = $term; + $this->_field = $field; + } + + /** + * Process modifier ('~') + * + * @param mixed $parameter + */ + public function processFuzzyProximityModifier($parameter = null) + { + $this->_fuzzyQuery = true; + + if ($parameter !== null) { + $this->_similarity = $parameter; + } else { + $this->_similarity = 0.5; + } + } + + /** + * Transform entry to a subquery + * + * @param string $encoding + * @return Zend_Search_Lucene_Search_Query + * @throws Zend_Search_Lucene_Search_QueryParserException + */ + public function getQuery($encoding) + { + if ($this->_fuzzyQuery) { + throw new Zend_Search_Lucene_Search_QueryParserException('Fuzzy search is not supported yet.'); + } + + if (strpos($this->_term, '?') !== false || strpos($this->_term, '*') !== false) { + throw new Zend_Search_Lucene_Search_QueryParserException('Wildcard queries are not supported yet.'); + } + + $tokens = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($this->_term, $encoding); + + if (count($tokens) == 0) { + return new Zend_Search_Lucene_Search_Query_Empty(); + } + + if (count($tokens) == 1) { + $term = new Zend_Search_Lucene_Index_Term($tokens[0]->getTermText(), $this->_field); + $query = new Zend_Search_Lucene_Search_Query_Term($term); + $query->setBoost($this->_boost); + + return $query; + } + + //It's not empty or one term query + $query = new Zend_Search_Lucene_Search_Query_MultiTerm(); + + /** + * @todo Process $token->getPositionIncrement() to support stemming, synonyms and other + * analizer design features + */ + foreach ($tokens as $token) { + $term = new Zend_Search_Lucene_Index_Term($token->getTermText(), $this->_field); + $query->addTerm($term, true); // all subterms are required + } + + $query->setBoost($this->_boost); + + return $query; + } +} diff --git a/search/Zend/Search/Lucene/Search/QueryHit.php b/search/Zend/Search/Lucene/Search/QueryHit.php index 19ab381fe46..12278ea4ad8 100644 --- a/search/Zend/Search/Lucene/Search/QueryHit.php +++ b/search/Zend/Search/Lucene/Search/QueryHit.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -24,14 +24,14 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_QueryHit { /** * Object handle of the index - * @var Zend_Search_Lucene + * @var Zend_Search_Lucene_Interface */ protected $_index = null; @@ -55,15 +55,15 @@ class Zend_Search_Lucene_Search_QueryHit /** - * Constructor - pass object handle of Zend_Search_Lucene index that produced + * Constructor - pass object handle of Zend_Search_Lucene_Interface index that produced * the hit so the document can be retrieved easily from the hit. * - * @param Zend_Search_Lucene $index + * @param Zend_Search_Lucene_Interface $index */ - public function __construct(Zend_Search_Lucene $index) + public function __construct(Zend_Search_Lucene_Interface $index) { - $this->_index = $index; + $this->_index = new Zend_Search_Lucene_Proxy($index); } @@ -98,7 +98,7 @@ class Zend_Search_Lucene_Search_QueryHit /** * Return the index object for this hit * - * @return Zend_Search_Lucene + * @return Zend_Search_Lucene_Interface */ public function getIndex() { diff --git a/search/Zend/Search/Lucene/Search/QueryLexer.php b/search/Zend/Search/Lucene/Search/QueryLexer.php new file mode 100644 index 00000000000..6b72110bfd1 --- /dev/null +++ b/search/Zend/Search/Lucene/Search/QueryLexer.php @@ -0,0 +1,508 @@ +dirroot.'/search/Zend/Search/Lucene/FSM.php'; + +/** Zend_Search_Lucene_Search_QueryParser */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryToken.php'; + +/** Zend_Search_Lucene_Exception */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; + +/** Zend_Search_Lucene_Search_QueryParserException */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Search + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Search_QueryLexer extends Zend_Search_Lucene_FSM +{ + /** State Machine states */ + const ST_WHITE_SPACE = 0; + const ST_SYNT_LEXEME = 1; + const ST_LEXEME = 2; + const ST_QUOTED_LEXEME = 3; + const ST_ESCAPED_CHAR = 4; + const ST_ESCAPED_QCHAR = 5; + const ST_LEXEME_MODIFIER = 6; + const ST_NUMBER = 7; + const ST_MANTISSA = 8; + const ST_ERROR = 9; + + /** Input symbols */ + const IN_WHITE_SPACE = 0; + const IN_SYNT_CHAR = 1; + const IN_LEXEME_MODIFIER = 2; + const IN_ESCAPE_CHAR = 3; + const IN_QUOTE = 4; + const IN_DECIMAL_POINT = 5; + const IN_ASCII_DIGIT = 6; + const IN_CHAR = 7; + const IN_MUTABLE_CHAR = 8; + + const QUERY_WHITE_SPACE_CHARS = " \n\r\t"; + const QUERY_SYNT_CHARS = ':()[]{}!|&'; + const QUERY_MUTABLE_CHARS = '+-'; + const QUERY_DOUBLECHARLEXEME_CHARS = '|&'; + const QUERY_LEXEMEMODIFIER_CHARS = '~^'; + const QUERY_ASCIIDIGITS_CHARS = '0123456789'; + + /** + * List of recognized lexemes + * + * @var array + */ + private $_lexemes; + + /** + * Query string (array of single- or non single-byte characters) + * + * @var array + */ + private $_queryString; + + /** + * Current position within a query string + * Used to create appropriate error messages + * + * @var integer + */ + private $_queryStringPosition; + + /** + * Recognized part of current lexeme + * + * @var string + */ + private $_currentLexeme; + + public function __construct() + { + parent::__construct( array(self::ST_WHITE_SPACE, + self::ST_SYNT_LEXEME, + self::ST_LEXEME, + self::ST_QUOTED_LEXEME, + self::ST_ESCAPED_CHAR, + self::ST_ESCAPED_QCHAR, + self::ST_LEXEME_MODIFIER, + self::ST_NUMBER, + self::ST_MANTISSA, + self::ST_ERROR), + array(self::IN_WHITE_SPACE, + self::IN_SYNT_CHAR, + self::IN_MUTABLE_CHAR, + self::IN_LEXEME_MODIFIER, + self::IN_ESCAPE_CHAR, + self::IN_QUOTE, + self::IN_DECIMAL_POINT, + self::IN_ASCII_DIGIT, + self::IN_CHAR)); + + + $lexemeModifierErrorAction = new Zend_Search_Lucene_FSMAction($this, 'lexModifierErrException'); + $quoteWithinLexemeErrorAction = new Zend_Search_Lucene_FSMAction($this, 'quoteWithinLexemeErrException'); + $wrongNumberErrorAction = new Zend_Search_Lucene_FSMAction($this, 'wrongNumberErrException'); + + + + $this->addRules(array( array(self::ST_WHITE_SPACE, self::IN_WHITE_SPACE, self::ST_WHITE_SPACE), + array(self::ST_WHITE_SPACE, self::IN_SYNT_CHAR, self::ST_SYNT_LEXEME), + array(self::ST_WHITE_SPACE, self::IN_MUTABLE_CHAR, self::ST_SYNT_LEXEME), + array(self::ST_WHITE_SPACE, self::IN_LEXEME_MODIFIER, self::ST_LEXEME_MODIFIER), + array(self::ST_WHITE_SPACE, self::IN_ESCAPE_CHAR, self::ST_ESCAPED_CHAR), + array(self::ST_WHITE_SPACE, self::IN_QUOTE, self::ST_QUOTED_LEXEME), + array(self::ST_WHITE_SPACE, self::IN_DECIMAL_POINT, self::ST_LEXEME), + array(self::ST_WHITE_SPACE, self::IN_ASCII_DIGIT, self::ST_LEXEME), + array(self::ST_WHITE_SPACE, self::IN_CHAR, self::ST_LEXEME) + )); + $this->addRules(array( array(self::ST_SYNT_LEXEME, self::IN_WHITE_SPACE, self::ST_WHITE_SPACE), + array(self::ST_SYNT_LEXEME, self::IN_SYNT_CHAR, self::ST_SYNT_LEXEME), + array(self::ST_SYNT_LEXEME, self::IN_MUTABLE_CHAR, self::ST_SYNT_LEXEME), + array(self::ST_SYNT_LEXEME, self::IN_LEXEME_MODIFIER, self::ST_LEXEME_MODIFIER), + array(self::ST_SYNT_LEXEME, self::IN_ESCAPE_CHAR, self::ST_ESCAPED_CHAR), + array(self::ST_SYNT_LEXEME, self::IN_QUOTE, self::ST_QUOTED_LEXEME), + array(self::ST_SYNT_LEXEME, self::IN_DECIMAL_POINT, self::ST_LEXEME), + array(self::ST_SYNT_LEXEME, self::IN_ASCII_DIGIT, self::ST_LEXEME), + array(self::ST_SYNT_LEXEME, self::IN_CHAR, self::ST_LEXEME) + )); + $this->addRules(array( array(self::ST_LEXEME, self::IN_WHITE_SPACE, self::ST_WHITE_SPACE), + array(self::ST_LEXEME, self::IN_SYNT_CHAR, self::ST_SYNT_LEXEME), + array(self::ST_LEXEME, self::IN_MUTABLE_CHAR, self::ST_LEXEME), + array(self::ST_LEXEME, self::IN_LEXEME_MODIFIER, self::ST_LEXEME_MODIFIER), + array(self::ST_LEXEME, self::IN_ESCAPE_CHAR, self::ST_ESCAPED_CHAR), + + // IN_QUOTE not allowed + array(self::ST_LEXEME, self::IN_QUOTE, self::ST_ERROR, $quoteWithinLexemeErrorAction), + + array(self::ST_LEXEME, self::IN_DECIMAL_POINT, self::ST_LEXEME), + array(self::ST_LEXEME, self::IN_ASCII_DIGIT, self::ST_LEXEME), + array(self::ST_LEXEME, self::IN_CHAR, self::ST_LEXEME) + )); + $this->addRules(array( array(self::ST_QUOTED_LEXEME, self::IN_WHITE_SPACE, self::ST_QUOTED_LEXEME), + array(self::ST_QUOTED_LEXEME, self::IN_SYNT_CHAR, self::ST_QUOTED_LEXEME), + array(self::ST_QUOTED_LEXEME, self::IN_MUTABLE_CHAR, self::ST_QUOTED_LEXEME), + array(self::ST_QUOTED_LEXEME, self::IN_LEXEME_MODIFIER, self::ST_QUOTED_LEXEME), + array(self::ST_QUOTED_LEXEME, self::IN_ESCAPE_CHAR, self::ST_ESCAPED_QCHAR), + array(self::ST_QUOTED_LEXEME, self::IN_QUOTE, self::ST_WHITE_SPACE), + array(self::ST_QUOTED_LEXEME, self::IN_DECIMAL_POINT, self::ST_QUOTED_LEXEME), + array(self::ST_QUOTED_LEXEME, self::IN_ASCII_DIGIT, self::ST_QUOTED_LEXEME), + array(self::ST_QUOTED_LEXEME, self::IN_CHAR, self::ST_QUOTED_LEXEME) + )); + $this->addRules(array( array(self::ST_ESCAPED_CHAR, self::IN_WHITE_SPACE, self::ST_LEXEME), + array(self::ST_ESCAPED_CHAR, self::IN_SYNT_CHAR, self::ST_LEXEME), + array(self::ST_ESCAPED_CHAR, self::IN_MUTABLE_CHAR, self::ST_LEXEME), + array(self::ST_ESCAPED_CHAR, self::IN_LEXEME_MODIFIER, self::ST_LEXEME), + array(self::ST_ESCAPED_CHAR, self::IN_ESCAPE_CHAR, self::ST_LEXEME), + array(self::ST_ESCAPED_CHAR, self::IN_QUOTE, self::ST_LEXEME), + array(self::ST_ESCAPED_CHAR, self::IN_DECIMAL_POINT, self::ST_LEXEME), + array(self::ST_ESCAPED_CHAR, self::IN_ASCII_DIGIT, self::ST_LEXEME), + array(self::ST_ESCAPED_CHAR, self::IN_CHAR, self::ST_LEXEME) + )); + $this->addRules(array( array(self::ST_ESCAPED_QCHAR, self::IN_WHITE_SPACE, self::ST_QUOTED_LEXEME), + array(self::ST_ESCAPED_QCHAR, self::IN_SYNT_CHAR, self::ST_QUOTED_LEXEME), + array(self::ST_ESCAPED_QCHAR, self::IN_MUTABLE_CHAR, self::ST_QUOTED_LEXEME), + array(self::ST_ESCAPED_QCHAR, self::IN_LEXEME_MODIFIER, self::ST_QUOTED_LEXEME), + array(self::ST_ESCAPED_QCHAR, self::IN_ESCAPE_CHAR, self::ST_QUOTED_LEXEME), + array(self::ST_ESCAPED_QCHAR, self::IN_QUOTE, self::ST_QUOTED_LEXEME), + array(self::ST_ESCAPED_QCHAR, self::IN_DECIMAL_POINT, self::ST_QUOTED_LEXEME), + array(self::ST_ESCAPED_QCHAR, self::IN_ASCII_DIGIT, self::ST_QUOTED_LEXEME), + array(self::ST_ESCAPED_QCHAR, self::IN_CHAR, self::ST_QUOTED_LEXEME) + )); + $this->addRules(array( array(self::ST_LEXEME_MODIFIER, self::IN_WHITE_SPACE, self::ST_WHITE_SPACE), + array(self::ST_LEXEME_MODIFIER, self::IN_SYNT_CHAR, self::ST_SYNT_LEXEME), + array(self::ST_LEXEME_MODIFIER, self::IN_MUTABLE_CHAR, self::ST_SYNT_LEXEME), + array(self::ST_LEXEME_MODIFIER, self::IN_LEXEME_MODIFIER, self::ST_LEXEME_MODIFIER), + + // IN_ESCAPE_CHAR not allowed + array(self::ST_LEXEME_MODIFIER, self::IN_ESCAPE_CHAR, self::ST_ERROR, $lexemeModifierErrorAction), + + // IN_QUOTE not allowed + array(self::ST_LEXEME_MODIFIER, self::IN_QUOTE, self::ST_ERROR, $lexemeModifierErrorAction), + + + array(self::ST_LEXEME_MODIFIER, self::IN_DECIMAL_POINT, self::ST_MANTISSA), + array(self::ST_LEXEME_MODIFIER, self::IN_ASCII_DIGIT, self::ST_NUMBER), + + // IN_CHAR not allowed + array(self::ST_LEXEME_MODIFIER, self::IN_CHAR, self::ST_ERROR, $lexemeModifierErrorAction), + )); + $this->addRules(array( array(self::ST_NUMBER, self::IN_WHITE_SPACE, self::ST_WHITE_SPACE), + array(self::ST_NUMBER, self::IN_SYNT_CHAR, self::ST_SYNT_LEXEME), + array(self::ST_NUMBER, self::IN_MUTABLE_CHAR, self::ST_SYNT_LEXEME), + array(self::ST_NUMBER, self::IN_LEXEME_MODIFIER, self::ST_LEXEME_MODIFIER), + + // IN_ESCAPE_CHAR not allowed + array(self::ST_NUMBER, self::IN_ESCAPE_CHAR, self::ST_ERROR, $wrongNumberErrorAction), + + // IN_QUOTE not allowed + array(self::ST_NUMBER, self::IN_QUOTE, self::ST_ERROR, $wrongNumberErrorAction), + + array(self::ST_NUMBER, self::IN_DECIMAL_POINT, self::ST_MANTISSA), + array(self::ST_NUMBER, self::IN_ASCII_DIGIT, self::ST_NUMBER), + + // IN_CHAR not allowed + array(self::ST_NUMBER, self::IN_CHAR, self::ST_ERROR, $wrongNumberErrorAction), + )); + $this->addRules(array( array(self::ST_MANTISSA, self::IN_WHITE_SPACE, self::ST_WHITE_SPACE), + array(self::ST_MANTISSA, self::IN_SYNT_CHAR, self::ST_SYNT_LEXEME), + array(self::ST_MANTISSA, self::IN_MUTABLE_CHAR, self::ST_SYNT_LEXEME), + array(self::ST_MANTISSA, self::IN_LEXEME_MODIFIER, self::ST_LEXEME_MODIFIER), + + // IN_ESCAPE_CHAR not allowed + array(self::ST_MANTISSA, self::IN_ESCAPE_CHAR, self::ST_ERROR, $wrongNumberErrorAction), + + // IN_QUOTE not allowed + array(self::ST_MANTISSA, self::IN_QUOTE, self::ST_ERROR, $wrongNumberErrorAction), + + // IN_DECIMAL_POINT not allowed + array(self::ST_MANTISSA, self::IN_DECIMAL_POINT, self::ST_ERROR, $wrongNumberErrorAction), + + array(self::ST_MANTISSA, self::IN_ASCII_DIGIT, self::ST_MANTISSA), + + // IN_CHAR not allowed + array(self::ST_MANTISSA, self::IN_CHAR, self::ST_ERROR, $wrongNumberErrorAction), + )); + + + /** Actions */ + $syntaxLexemeAction = new Zend_Search_Lucene_FSMAction($this, 'addQuerySyntaxLexeme'); + $lexemeModifierAction = new Zend_Search_Lucene_FSMAction($this, 'addLexemeModifier'); + $addLexemeAction = new Zend_Search_Lucene_FSMAction($this, 'addLexeme'); + $addQuotedLexemeAction = new Zend_Search_Lucene_FSMAction($this, 'addQuotedLexeme'); + $addNumberLexemeAction = new Zend_Search_Lucene_FSMAction($this, 'addNumberLexeme'); + $addLexemeCharAction = new Zend_Search_Lucene_FSMAction($this, 'addLexemeChar'); + + + /** Syntax lexeme */ + $this->addEntryAction(self::ST_SYNT_LEXEME, $syntaxLexemeAction); + // Two lexemes in succession + $this->addTransitionAction(self::ST_SYNT_LEXEME, self::ST_SYNT_LEXEME, $syntaxLexemeAction); + + + /** Lexeme */ + $this->addEntryAction(self::ST_LEXEME, $addLexemeCharAction); + $this->addTransitionAction(self::ST_LEXEME, self::ST_LEXEME, $addLexemeCharAction); + // ST_ESCAPED_CHAR => ST_LEXEME transition is covered by ST_LEXEME entry action + + $this->addTransitionAction(self::ST_LEXEME, self::ST_WHITE_SPACE, $addLexemeAction); + $this->addTransitionAction(self::ST_LEXEME, self::ST_SYNT_LEXEME, $addLexemeAction); + $this->addTransitionAction(self::ST_LEXEME, self::ST_QUOTED_LEXEME, $addLexemeAction); + $this->addTransitionAction(self::ST_LEXEME, self::ST_LEXEME_MODIFIER, $addLexemeAction); + $this->addTransitionAction(self::ST_LEXEME, self::ST_NUMBER, $addLexemeAction); + $this->addTransitionAction(self::ST_LEXEME, self::ST_MANTISSA, $addLexemeAction); + + + /** Quoted lexeme */ + // We don't need entry action (skeep quote) + $this->addTransitionAction(self::ST_QUOTED_LEXEME, self::ST_QUOTED_LEXEME, $addLexemeCharAction); + $this->addTransitionAction(self::ST_ESCAPED_QCHAR, self::ST_QUOTED_LEXEME, $addLexemeCharAction); + // Closing quote changes state to the ST_WHITE_SPACE other states are not used + $this->addTransitionAction(self::ST_QUOTED_LEXEME, self::ST_WHITE_SPACE, $addQuotedLexemeAction); + + + /** Lexeme modifier */ + $this->addEntryAction(self::ST_LEXEME_MODIFIER, $lexemeModifierAction); + + + /** Number */ + $this->addEntryAction(self::ST_NUMBER, $addLexemeCharAction); + $this->addEntryAction(self::ST_MANTISSA, $addLexemeCharAction); + $this->addTransitionAction(self::ST_NUMBER, self::ST_NUMBER, $addLexemeCharAction); + // ST_NUMBER => ST_MANTISSA transition is covered by ST_MANTISSA entry action + $this->addTransitionAction(self::ST_MANTISSA, self::ST_MANTISSA, $addLexemeCharAction); + + $this->addTransitionAction(self::ST_NUMBER, self::ST_WHITE_SPACE, $addNumberLexemeAction); + $this->addTransitionAction(self::ST_NUMBER, self::ST_SYNT_LEXEME, $addNumberLexemeAction); + $this->addTransitionAction(self::ST_NUMBER, self::ST_LEXEME_MODIFIER, $addNumberLexemeAction); + $this->addTransitionAction(self::ST_MANTISSA, self::ST_WHITE_SPACE, $addNumberLexemeAction); + $this->addTransitionAction(self::ST_MANTISSA, self::ST_SYNT_LEXEME, $addNumberLexemeAction); + $this->addTransitionAction(self::ST_MANTISSA, self::ST_LEXEME_MODIFIER, $addNumberLexemeAction); + } + + + + + /** + * Translate input char to an input symbol of state machine + * + * @param string $char + * @return integer + */ + private function _translateInput($char) + { + if (strpos(self::QUERY_WHITE_SPACE_CHARS, $char) !== false) { return self::IN_WHITE_SPACE; + } else if (strpos(self::QUERY_SYNT_CHARS, $char) !== false) { return self::IN_SYNT_CHAR; + } else if (strpos(self::QUERY_MUTABLE_CHARS, $char) !== false) { return self::IN_MUTABLE_CHAR; + } else if (strpos(self::QUERY_LEXEMEMODIFIER_CHARS, $char) !== false) { return self::IN_LEXEME_MODIFIER; + } else if (strpos(self::QUERY_ASCIIDIGITS_CHARS, $char) !== false) { return self::IN_ASCII_DIGIT; + } else if ($char === '"' ) { return self::IN_QUOTE; + } else if ($char === '.' ) { return self::IN_DECIMAL_POINT; + } else if ($char === '\\') { return self::IN_ESCAPE_CHAR; + } else { return self::IN_CHAR; + } + } + + + /** + * This method is used to tokenize query string into lexemes + * + * @param string $inputString + * @param string $encoding + * @return array + * @throws Zend_Search_Lucene_Search_QueryParserException + */ + public function tokenize($inputString, $encoding) + { + $this->reset(); + + $this->_lexemes = array(); + $this->_queryString = array(); + + $strLength = iconv_strlen($inputString, $encoding); + + // Workaround for iconv_substr bug + $inputString .= ' '; + + for ($count = 0; $count < $strLength; $count++) { + $this->_queryString[$count] = iconv_substr($inputString, $count, 1, $encoding); + } + + for ($this->_queryStringPosition = 0; + $this->_queryStringPosition < count($this->_queryString); + $this->_queryStringPosition++) { + $this->process($this->_translateInput($this->_queryString[$this->_queryStringPosition])); + } + + $this->process(self::IN_WHITE_SPACE); + + if ($this->getState() != self::ST_WHITE_SPACE) { + throw new Zend_Search_Lucene_Search_QueryParserException('Unexpected end of query'); + } + + $this->_queryString = null; + + return $this->_lexemes; + } + + + + /********************************************************************* + * Actions implementation + * + * Actions affect on recognized lexemes list + *********************************************************************/ + + /** + * Add query syntax lexeme + * + * @throws Zend_Search_Lucene_Search_QueryParserException + */ + public function addQuerySyntaxLexeme() + { + $lexeme = $this->_queryString[$this->_queryStringPosition]; + + // Process two char lexemes + if (strpos(self::QUERY_DOUBLECHARLEXEME_CHARS, $lexeme) !== false) { + // increase current position in a query string + $this->_queryStringPosition++; + + // check, + if ($this->_queryStringPosition == count($this->_queryString) || + $this->_queryString[$this->_queryStringPosition] != $lexeme) { + throw new Zend_Search_Lucene_Search_QueryParserException('Two chars lexeme expected. ' . $this->_positionMsg()); + } + + // duplicate character + $lexeme .= $lexeme; + } + + $token = new Zend_Search_Lucene_Search_QueryToken( + Zend_Search_Lucene_Search_QueryToken::TC_SYNTAX_ELEMENT, + $lexeme, + $this->_queryStringPosition); + + // Skip this lexeme if it's a field indicator ':' and treat previous as 'field' instead of 'word' + if ($token->type == Zend_Search_Lucene_Search_QueryToken::TT_FIELD_INDICATOR) { + $token = array_pop($this->_lexemes); + if ($token === null || $token->type != Zend_Search_Lucene_Search_QueryToken::TT_WORD) { + throw new Zend_Search_Lucene_Search_QueryParserException('Field mark \':\' must follow field name. ' . $this->_positionMsg()); + } + + $token->type = Zend_Search_Lucene_Search_QueryToken::TT_FIELD; + } + + $this->_lexemes[] = $token; + } + + /** + * Add lexeme modifier + */ + public function addLexemeModifier() + { + $this->_lexemes[] = new Zend_Search_Lucene_Search_QueryToken( + Zend_Search_Lucene_Search_QueryToken::TC_SYNTAX_ELEMENT, + $this->_queryString[$this->_queryStringPosition], + $this->_queryStringPosition); + } + + + /** + * Add lexeme + */ + public function addLexeme() + { + $this->_lexemes[] = new Zend_Search_Lucene_Search_QueryToken( + Zend_Search_Lucene_Search_QueryToken::TC_WORD, + $this->_currentLexeme, + $this->_queryStringPosition - 1); + + $this->_currentLexeme = ''; + } + + /** + * Add quoted lexeme + */ + public function addQuotedLexeme() + { + $this->_lexemes[] = new Zend_Search_Lucene_Search_QueryToken( + Zend_Search_Lucene_Search_QueryToken::TC_PHRASE, + $this->_currentLexeme, + $this->_queryStringPosition); + + $this->_currentLexeme = ''; + } + + /** + * Add number lexeme + */ + public function addNumberLexeme() + { + $this->_lexemes[] = new Zend_Search_Lucene_Search_QueryToken( + Zend_Search_Lucene_Search_QueryToken::TC_NUMBER, + $this->_currentLexeme, + $this->_queryStringPosition - 1); + $this->_currentLexeme = ''; + } + + /** + * Extend lexeme by one char + */ + public function addLexemeChar() + { + $this->_currentLexeme .= $this->_queryString[$this->_queryStringPosition]; + } + + + /** + * Position message + * + * @return string + */ + private function _positionMsg() + { + return 'Position is ' . $this->_queryStringPosition . '.'; + } + + + /********************************************************************* + * Syntax errors actions + *********************************************************************/ + public function lexModifierErrException() + { + throw new Zend_Search_Lucene_Search_QueryParserException('Lexeme modifier character can be followed only by number, white space or query syntax element. ' . $this->_positionMsg()); + } + public function quoteWithinLexemeErrException() + { + throw new Zend_Search_Lucene_Search_QueryParserException('Quote within lexeme must be escaped by \'\\\' char. ' . $this->_positionMsg()); + } + public function wrongNumberErrException() + { + throw new Zend_Search_Lucene_Search_QueryParserException('Wrong number syntax.' . $this->_positionMsg()); + } +} + diff --git a/search/Zend/Search/Lucene/Search/QueryParser.php b/search/Zend/Search/Lucene/Search/QueryParser.php index 63b6497e054..1a3d5712de8 100644 --- a/search/Zend/Search/Lucene/Search/QueryParser.php +++ b/search/Zend/Search/Lucene/Search/QueryParser.php @@ -15,128 +15,507 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ -/** Zend_Search_Lucene_Search_QueryTokenizer */ -require_once 'Zend/Search/Lucene/Search/QueryTokenizer.php'; - /** Zend_Search_Lucene_Index_Term */ -require_once 'Zend/Search/Lucene/Index/Term.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php'; /** Zend_Search_Lucene_Search_Query_Term */ -require_once 'Zend/Search/Lucene/Search/Query/Term.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Term.php'; /** Zend_Search_Lucene_Search_Query_MultiTerm */ -require_once 'Zend/Search/Lucene/Search/Query/MultiTerm.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/MultiTerm.php'; + +/** Zend_Search_Lucene_Search_Query_Boolean */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Boolean.php'; /** Zend_Search_Lucene_Search_Query_Phrase */ -require_once 'Zend/Search/Lucene/Search/Query/Phrase.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Phrase.php'; +/** Zend_Search_Lucene_Search_Query_Empty */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Empty.php'; + + +/** Zend_Search_Lucene_Search_QueryLexer */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryLexer.php'; + +/** Zend_Search_Lucene_Search_QueryParserContext */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserContext.php'; + + +/** Zend_Search_Lucene_FSM */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/FSM.php'; /** Zend_Search_Lucene_Exception */ -require_once 'Zend/Search/Lucene/Exception.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; + +/** Zend_Search_Lucene_Search_QueryParserException */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ -class Zend_Search_Lucene_Search_QueryParser +class Zend_Search_Lucene_Search_QueryParser extends Zend_Search_Lucene_FSM { + /** + * Parser instance + * + * @var Zend_Search_Lucene_Search_QueryParser + */ + private static $_instance = null; + /** - * Parses a query string, returning a Zend_Search_Lucene_Search_Query + * Query lexer + * + * @var Zend_Search_Lucene_Search_QueryLexer + */ + private $_lexer; + + /** + * Tokens list + * Array of Zend_Search_Lucene_Search_QueryToken objects + * + * @var array + */ + private $_tokens; + + /** + * Current token + * + * @var integer|string + */ + private $_currentToken; + + /** + * Last token + * + * It can be processed within FSM states, but this addirional state simplifies FSM + * + * @var Zend_Search_Lucene_Search_QueryToken + */ + private $_lastToken = null; + + /** + * Range query first term + * + * @var string + */ + private $_rqFirstTerm = null; + + /** + * Current query parser context + * + * @var Zend_Search_Lucene_Search_QueryParserContext + */ + private $_context; + + /** + * Context stack + * + * @var array + */ + private $_contextStack; + + /** + * Query string encoding + * + * @var string + */ + private $_encoding; + + /** + * Query string default encoding + * + * @var string + */ + private $_defaultEncoding = ''; + + + /** + * Boolean operators constants + */ + const B_OR = 0; + const B_AND = 1; + + /** + * Default boolean queries operator + * + * @var integer + */ + private $_defaultOperator = self::B_OR; + + + /** Query parser State Machine states */ + const ST_COMMON_QUERY_ELEMENT = 0; // Terms, phrases, operators + const ST_CLOSEDINT_RQ_START = 1; // Range query start (closed interval) - '[' + const ST_CLOSEDINT_RQ_FIRST_TERM = 2; // First term in '[term1 to term2]' construction + const ST_CLOSEDINT_RQ_TO_TERM = 3; // 'TO' lexeme in '[term1 to term2]' construction + const ST_CLOSEDINT_RQ_LAST_TERM = 4; // Second term in '[term1 to term2]' construction + const ST_CLOSEDINT_RQ_END = 5; // Range query end (closed interval) - ']' + const ST_OPENEDINT_RQ_START = 6; // Range query start (opened interval) - '{' + const ST_OPENEDINT_RQ_FIRST_TERM = 7; // First term in '{term1 to term2}' construction + const ST_OPENEDINT_RQ_TO_TERM = 8; // 'TO' lexeme in '{term1 to term2}' construction + const ST_OPENEDINT_RQ_LAST_TERM = 9; // Second term in '{term1 to term2}' construction + const ST_OPENEDINT_RQ_END = 10; // Range query end (opened interval) - '}' + + /** + * Parser constructor + */ + public function __construct() + { + parent::__construct(array(self::ST_COMMON_QUERY_ELEMENT, + self::ST_CLOSEDINT_RQ_START, + self::ST_CLOSEDINT_RQ_FIRST_TERM, + self::ST_CLOSEDINT_RQ_TO_TERM, + self::ST_CLOSEDINT_RQ_LAST_TERM, + self::ST_CLOSEDINT_RQ_END, + self::ST_OPENEDINT_RQ_START, + self::ST_OPENEDINT_RQ_FIRST_TERM, + self::ST_OPENEDINT_RQ_TO_TERM, + self::ST_OPENEDINT_RQ_LAST_TERM, + self::ST_OPENEDINT_RQ_END + ), + Zend_Search_Lucene_Search_QueryToken::getTypes()); + + $this->addRules( + array(array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_WORD, self::ST_COMMON_QUERY_ELEMENT), + array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_PHRASE, self::ST_COMMON_QUERY_ELEMENT), + array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_FIELD, self::ST_COMMON_QUERY_ELEMENT), + array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_REQUIRED, self::ST_COMMON_QUERY_ELEMENT), + array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_PROHIBITED, self::ST_COMMON_QUERY_ELEMENT), + array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_FUZZY_PROX_MARK, self::ST_COMMON_QUERY_ELEMENT), + array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_BOOSTING_MARK, self::ST_COMMON_QUERY_ELEMENT), + array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_RANGE_INCL_START, self::ST_CLOSEDINT_RQ_START), + array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_RANGE_EXCL_START, self::ST_OPENEDINT_RQ_START), + array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_SUBQUERY_START, self::ST_COMMON_QUERY_ELEMENT), + array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_SUBQUERY_END, self::ST_COMMON_QUERY_ELEMENT), + array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_AND_LEXEME, self::ST_COMMON_QUERY_ELEMENT), + array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_OR_LEXEME, self::ST_COMMON_QUERY_ELEMENT), + array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_NOT_LEXEME, self::ST_COMMON_QUERY_ELEMENT), + array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_NUMBER, self::ST_COMMON_QUERY_ELEMENT) + )); + $this->addRules( + array(array(self::ST_CLOSEDINT_RQ_START, Zend_Search_Lucene_Search_QueryToken::TT_WORD, self::ST_CLOSEDINT_RQ_FIRST_TERM), + array(self::ST_CLOSEDINT_RQ_FIRST_TERM, Zend_Search_Lucene_Search_QueryToken::TT_TO_LEXEME, self::ST_CLOSEDINT_RQ_TO_TERM), + array(self::ST_CLOSEDINT_RQ_TO_TERM, Zend_Search_Lucene_Search_QueryToken::TT_WORD, self::ST_CLOSEDINT_RQ_LAST_TERM), + array(self::ST_CLOSEDINT_RQ_LAST_TERM, Zend_Search_Lucene_Search_QueryToken::TT_RANGE_INCL_END, self::ST_COMMON_QUERY_ELEMENT) + )); + $this->addRules( + array(array(self::ST_OPENEDINT_RQ_START, Zend_Search_Lucene_Search_QueryToken::TT_WORD, self::ST_OPENEDINT_RQ_FIRST_TERM), + array(self::ST_OPENEDINT_RQ_FIRST_TERM, Zend_Search_Lucene_Search_QueryToken::TT_TO_LEXEME, self::ST_OPENEDINT_RQ_TO_TERM), + array(self::ST_OPENEDINT_RQ_TO_TERM, Zend_Search_Lucene_Search_QueryToken::TT_WORD, self::ST_OPENEDINT_RQ_LAST_TERM), + array(self::ST_OPENEDINT_RQ_LAST_TERM, Zend_Search_Lucene_Search_QueryToken::TT_RANGE_EXCL_END, self::ST_COMMON_QUERY_ELEMENT) + )); + + + + $addTermEntryAction = new Zend_Search_Lucene_FSMAction($this, 'addTermEntry'); + $addPhraseEntryAction = new Zend_Search_Lucene_FSMAction($this, 'addPhraseEntry'); + $setFieldAction = new Zend_Search_Lucene_FSMAction($this, 'setField'); + $setSignAction = new Zend_Search_Lucene_FSMAction($this, 'setSign'); + $setFuzzyProxAction = new Zend_Search_Lucene_FSMAction($this, 'processFuzzyProximityModifier'); + $processModifierParameterAction = new Zend_Search_Lucene_FSMAction($this, 'processModifierParameter'); + $subqueryStartAction = new Zend_Search_Lucene_FSMAction($this, 'subqueryStart'); + $subqueryEndAction = new Zend_Search_Lucene_FSMAction($this, 'subqueryEnd'); + $logicalOperatorAction = new Zend_Search_Lucene_FSMAction($this, 'logicalOperator'); + $openedRQFirstTermAction = new Zend_Search_Lucene_FSMAction($this, 'openedRQFirstTerm'); + $openedRQLastTermAction = new Zend_Search_Lucene_FSMAction($this, 'openedRQLastTerm'); + $closedRQFirstTermAction = new Zend_Search_Lucene_FSMAction($this, 'closedRQFirstTerm'); + $closedRQLastTermAction = new Zend_Search_Lucene_FSMAction($this, 'closedRQLastTerm'); + + + $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_WORD, $addTermEntryAction); + $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_PHRASE, $addPhraseEntryAction); + $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_FIELD, $setFieldAction); + $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_REQUIRED, $setSignAction); + $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_PROHIBITED, $setSignAction); + $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_FUZZY_PROX_MARK, $setFuzzyProxAction); + $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_NUMBER, $processModifierParameterAction); + $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_SUBQUERY_START, $subqueryStartAction); + $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_SUBQUERY_END, $subqueryEndAction); + $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_AND_LEXEME, $logicalOperatorAction); + $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_OR_LEXEME, $logicalOperatorAction); + $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_NOT_LEXEME, $logicalOperatorAction); + + $this->addEntryAction(self::ST_OPENEDINT_RQ_FIRST_TERM, $openedRQFirstTermAction); + $this->addEntryAction(self::ST_OPENEDINT_RQ_LAST_TERM, $openedRQLastTermAction); + $this->addEntryAction(self::ST_CLOSEDINT_RQ_FIRST_TERM, $closedRQFirstTermAction); + $this->addEntryAction(self::ST_CLOSEDINT_RQ_LAST_TERM, $closedRQLastTermAction); + + + + $this->_lexer = new Zend_Search_Lucene_Search_QueryLexer(); + } + + + /** + * Set query string default encoding + * + * @param string $encoding + */ + public static function setDefaultEncoding($encoding) + { + if (self::$_instance === null) { + self::$_instance = new Zend_Search_Lucene_Search_QueryParser(); + } + + self::$_instance->_defaultEncoding = $encoding; + } + + /** + * Get query string default encoding + * + * @return string + */ + public static function getDefaultEncoding() + { + if (self::$_instance === null) { + self::$_instance = new Zend_Search_Lucene_Search_QueryParser(); + } + + return self::$_instance->_defaultEncoding; + } + + /** + * Set default boolean operator + * + * @param integer $operator + */ + public static function setDefaultOperator($operator) + { + if (self::$_instance === null) { + self::$_instance = new Zend_Search_Lucene_Search_QueryParser(); + } + + self::$_instance->_defaultOperator = $operator; + } + + /** + * Get default boolean operator + * + * @return integer + */ + public static function getDefaultOperator() + { + if (self::$_instance === null) { + self::$_instance = new Zend_Search_Lucene_Search_QueryParser(); + } + + return self::$_instance->_defaultOperator; + } + + /** + * Parses a query string * * @param string $strQuery + * @param string $encoding * @return Zend_Search_Lucene_Search_Query + * @throws Zend_Search_Lucene_Search_QueryParserException */ - static public function parse($strQuery) + public static function parse($strQuery, $encoding = null) { - $tokens = new Zend_Search_Lucene_Search_QueryTokenizer($strQuery); + if (self::$_instance === null) { + self::$_instance = new Zend_Search_Lucene_Search_QueryParser(); + } + + self::$_instance->_encoding = ($encoding !== null) ? $encoding : self::$_instance->_defaultEncoding; + self::$_instance->_lastToken = null; + self::$_instance->_context = new Zend_Search_Lucene_Search_QueryParserContext(self::$_instance->_encoding); + self::$_instance->_contextStack = array(); + self::$_instance->_tokens = self::$_instance->_lexer->tokenize($strQuery, self::$_instance->_encoding); // Empty query - if (!$tokens->count()) { - throw new Zend_Search_Lucene_Exception('Syntax error: query string cannot be empty.'); + if (count(self::$_instance->_tokens) == 0) { + return new Zend_Search_Lucene_Search_Query_Empty(); } - // Term query - if ($tokens->count() == 1) { - if ($tokens->current()->type == Zend_Search_Lucene_Search_QueryToken::TOKTYPE_WORD) { - return new Zend_Search_Lucene_Search_Query_Term(new Zend_Search_Lucene_Index_Term($tokens->current()->text, 'contents')); - } else { - throw new Zend_Search_Lucene_Exception('Syntax error: query string must contain at least one word.'); + + foreach (self::$_instance->_tokens as $token) { + try { + self::$_instance->_currentToken = $token; + self::$_instance->process($token->type); + + self::$_instance->_lastToken = $token; + } catch (Exception $e) { + if (strpos($e->getMessage(), 'There is no any rule for') !== false) { + throw new Zend_Search_Lucene_Search_QueryParserException( 'Syntax error at char position ' . $token->position . '.' ); + } + + throw $e; } } - - /** - * MultiTerm Query - * - * Process each token that was returned by the tokenizer. - */ - $terms = array(); - $signs = array(); - $prevToken = null; - $openBrackets = 0; - $field = 'contents'; - foreach ($tokens as $token) { - switch ($token->type) { - case Zend_Search_Lucene_Search_QueryToken::TOKTYPE_WORD: - $terms[] = new Zend_Search_Lucene_Index_Term($token->text, $field); - $field = 'contents'; - if ($prevToken !== null && - $prevToken->type == Zend_Search_Lucene_Search_QueryToken::TOKTYPE_SIGN) { - if ($prevToken->text == "+") { - $signs[] = true; - } else { - $signs[] = false; - } - } else { - $signs[] = null; - } - break; - case Zend_Search_Lucene_Search_QueryToken::TOKTYPE_SIGN: - if ($prevToken !== null && - $prevToken->type == Zend_Search_Lucene_Search_QueryToken::TOKTYPE_SIGN) { - throw new Zend_Search_Lucene_Exception('Syntax error: sign operator must be followed by a word.'); - } - break; - case Zend_Search_Lucene_Search_QueryToken::TOKTYPE_FIELD: - $field = $token->text; - // let previous token to be signed as next $prevToken - $token = $prevToken; - break; - case Zend_Search_Lucene_Search_QueryToken::TOKTYPE_BRACKET: - $token->text=='(' ? $openBrackets++ : $openBrackets--; - } - $prevToken = $token; + if (count(self::$_instance->_contextStack) != 0) { + throw new Zend_Search_Lucene_Search_QueryParserException('Syntax Error: mismatched parentheses, every opening must have closing.' ); } - // Finish up parsing: check the last token in the query for an opening sign or parenthesis. - if ($prevToken->type == Zend_Search_Lucene_Search_QueryToken::TOKTYPE_SIGN) { - throw new Zend_Search_Lucene_Exception('Syntax Error: sign operator must be followed by a word.'); + return self::$_instance->_context->getQuery(); + } + + + /********************************************************************* + * Actions implementation + * + * Actions affect on recognized lexemes list + *********************************************************************/ + + /** + * Add term to a query + */ + public function addTermEntry() + { + $entry = new Zend_Search_Lucene_Search_QueryEntry_Term($this->_currentToken->text, $this->_context->getField()); + $this->_context->addEntry($entry); + } + + /** + * Add phrase to a query + */ + public function addPhraseEntry() + { + $entry = new Zend_Search_Lucene_Search_QueryEntry_Phrase($this->_currentToken->text, $this->_context->getField()); + $this->_context->addEntry($entry); + } + + /** + * Set entry field + */ + public function setField() + { + $this->_context->setNextEntryField($this->_currentToken->text); + } + + /** + * Set entry sign + */ + public function setSign() + { + $this->_context->setNextEntrySign($this->_currentToken->type); + } + + + /** + * Process fuzzy search/proximity modifier - '~' + */ + public function processFuzzyProximityModifier() + { + $this->_context->processFuzzyProximityModifier(); + } + + /** + * Process modifier parameter + * + * @throws Zend_Search_Lucene_Exception + */ + public function processModifierParameter() + { + if ($this->_lastToken === null) { + throw new Zend_Search_Lucene_Search_QueryParserException('Lexeme modifier parameter must follow lexeme modifier. Char position 0.' ); } - // Finish up parsing: check that every opening bracket has a matching closing bracket. - if ($openBrackets != 0) { - throw new Zend_Search_Lucene_Exception('Syntax Error: mismatched parentheses, every opening must have closing.'); - } + switch ($this->_lastToken->type) { + case Zend_Search_Lucene_Search_QueryToken::TT_FUZZY_PROX_MARK: + $this->_context->processFuzzyProximityModifier($this->_currentToken->text); + break; + + case Zend_Search_Lucene_Search_QueryToken::TT_BOOSTING_MARK: + $this->_context->boost($this->_currentToken->text); + break; - switch (count($terms)) { - case 0: - throw new Zend_Search_Lucene_Exception('Syntax error: bad term count.'); - case 1: - return new Zend_Search_Lucene_Search_Query_Term($terms[0],$signs[0] !== false); default: - return new Zend_Search_Lucene_Search_Query_MultiTerm($terms,$signs); + // It's not a user input exception + throw new Zend_Search_Lucene_Exception('Lexeme modifier parameter must follow lexeme modifier. Char position .' ); } } + + /** + * Start subquery + */ + public function subqueryStart() + { + $this->_contextStack[] = $this->_context; + $this->_context = new Zend_Search_Lucene_Search_QueryParserContext($this->_encoding, $this->_context->getField()); + } + + /** + * End subquery + */ + public function subqueryEnd() + { + if (count($this->_contextStack) == 0) { + throw new Zend_Search_Lucene_Search_QueryParserException('Syntax Error: mismatched parentheses, every opening must have closing. Char position ' . $this->_currentToken->position . '.' ); + } + + $query = $this->_context->getQuery(); + $this->_context = array_pop($this->_contextStack); + + $this->_context->addEntry(new Zend_Search_Lucene_Search_QueryEntry_Subquery($query)); + } + + /** + * Process logical operator + */ + public function logicalOperator() + { + $this->_context->addLogicalOperator($this->_currentToken->type); + } + + /** + * Process first range query term (opened interval) + */ + public function openedRQFirstTerm() + { + $this->_rqFirstTerm = $this->_currentToken->text; + } + + /** + * Process last range query term (opened interval) + * + * @throws Zend_Search_Lucene_Search_QueryParserException + */ + public function openedRQLastTerm() + { + throw new Zend_Search_Lucene_Search_QueryParserException('Range queries are not supported yet.'); + + // $firstTerm = new Zend_Search_Lucene_Index_Term($this->_rqFirstTerm, $this->_context->getField()); + // $lastTerm = new Zend_Search_Lucene_Index_Term($this->_currentToken->text, $this->_context->getField()); + + // $query = new Zend_Search_Lucene_Search_Query_Range($firstTerm, $lastTerm, false); + // $this->_context->addentry($query); + } + + /** + * Process first range query term (closed interval) + */ + public function closedRQFirstTerm() + { + $this->_rqFirstTerm = $this->_currentToken->text; + } + + /** + * Process last range query term (closed interval) + * + * @throws Zend_Search_Lucene_Search_QueryParserException + */ + public function closedRQLastTerm() + { + throw new Zend_Search_Lucene_Search_QueryParserException('Range queries are not supported yet.'); + + // $firstTerm = new Zend_Search_Lucene_Index_Term($this->_rqFirstTerm, $this->_context->getField()); + // $lastTerm = new Zend_Search_Lucene_Index_Term($this->_currentToken->text, $this->_context->getField()); + + // $query = new Zend_Search_Lucene_Search_Query_Range($firstTerm, $lastTerm, true); + // $this->_context->addentry($query); + } } diff --git a/search/Zend/Search/Lucene/Search/QueryParserContext.php b/search/Zend/Search/Lucene/Search/QueryParserContext.php new file mode 100644 index 00000000000..9d172c69fa5 --- /dev/null +++ b/search/Zend/Search/Lucene/Search/QueryParserContext.php @@ -0,0 +1,416 @@ +dirroot.'/search/Zend/Search/Lucene/FSM.php'; + + +/** Zend_Search_Lucene_Index_Term */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php'; + +/** Zend_Search_Lucene_Search_QueryToken */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryToken.php'; + +/** Zend_Search_Lucene_Search_Query_Term */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Term.php'; + +/** Zend_Search_Lucene_Search_Query_MultiTerm */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/MultiTerm.php'; + +/** Zend_Search_Lucene_Search_Query_Boolean */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Boolean.php'; + +/** Zend_Search_Lucene_Search_Query_Phrase */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Phrase.php'; + +/** Zend_Search_Lucene_Exception */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; + +/** Zend_Search_Lucene_Search_QueryParserException */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php'; + +/** Zend_Search_Lucene_Search_BooleanExpressionRecognizer */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/BooleanExpressionRecognizer.php'; + +/** Zend_Search_Lucene_Search_QueryEntry */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Search + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Search_QueryParserContext +{ + /** + * Default field for the context. + * + * null means, that term should be searched through all fields + * Zend_Search_Lucene_Search_Query::rewriteQuery($index) transletes such queries to several + * + * @var string|null + */ + private $_defaultField; + + /** + * Field specified for next entry + * + * @var string + */ + private $_nextEntryField = null; + + /** + * True means, that term is required. + * False means, that term is prohibited. + * null means, that term is neither prohibited, nor required + * + * @var boolean + */ + private $_nextEntrySign = null; + + + /** + * Entries grouping mode + */ + const GM_SIGNS = 0; // Signs mode: '+term1 term2 -term3 +(subquery1) -(subquery2)' + const GM_BOOLEAN = 1; // Boolean operators mode: 'term1 and term2 or (subquery1) and not (subquery2)' + + /** + * Grouping mode + * + * @var integer + */ + private $_mode = null; + + /** + * Entries signs. + * Used in GM_SIGNS grouping mode + * + * @var arrays + */ + private $_signs = array(); + + /** + * Query entries + * Each entry is a Zend_Search_Lucene_Search_QueryEntry object or + * boolean operator (Zend_Search_Lucene_Search_QueryToken class constant) + * + * @var array + */ + private $_entries = array(); + + /** + * Query string encoding + * + * @var string + */ + private $_encoding; + + + /** + * Context object constructor + * + * @param string $encoding + * @param string|null $defaultField + */ + public function __construct($encoding, $defaultField = null) + { + $this->_encoding = $encoding; + $this->_defaultField = $defaultField; + } + + + /** + * Get context default field + * + * @return string|null + */ + public function getField() + { + return ($this->_nextEntryField !== null) ? $this->_nextEntryField : $this->_defaultField; + } + + /** + * Set field for next entry + * + * @param string $field + */ + public function setNextEntryField($field) + { + $this->_nextEntryField = $field; + } + + + /** + * Set sign for next entry + * + * @param integer $sign + * @throws Zend_Search_Lucene_Exception + */ + public function setNextEntrySign($sign) + { + if ($this->_mode === self::GM_BOOLEAN) { + throw new Zend_Search_Lucene_Search_QueryParserException('It\'s not allowed to mix boolean and signs styles in the same subquery.'); + } + + $this->_mode = self::GM_SIGNS; + + if ($sign == Zend_Search_Lucene_Search_QueryToken::TT_REQUIRED) { + $this->_nextEntrySign = true; + } else if ($sign == Zend_Search_Lucene_Search_QueryToken::TT_PROHIBITED) { + $this->_nextEntrySign = false; + } else { + throw new Zend_Search_Lucene_Exception('Unrecognized sign type.'); + } + } + + + /** + * Add entry to a query + * + * @param Zend_Search_Lucene_Search_QueryEntry $entry + */ + public function addEntry(Zend_Search_Lucene_Search_QueryEntry $entry) + { + if ($this->_mode !== self::GM_BOOLEAN) { + $this->_signs[] = $this->_nextEntrySign; + } + + $this->_entries[] = $entry; + + $this->_nextEntryField = null; + $this->_nextEntrySign = null; + } + + + /** + * Process fuzzy search or proximity search modifier + * + * @throws Zend_Search_Lucene_Search_QueryParserException + */ + public function processFuzzyProximityModifier($parameter = null) + { + // Check, that modifier has came just after word or phrase + if ($this->_nextEntryField !== null || $this->_nextEntrySign !== null) { + throw new Zend_Search_Lucene_Search_QueryParserException('\'~\' modifier must follow word or phrase.'); + } + + $lastEntry = array_pop($this->_entries); + + if (!$lastEntry instanceof Zend_Search_Lucene_Search_QueryEntry) { + // there are no entries or last entry is boolean operator + throw new Zend_Search_Lucene_Search_QueryParserException('\'~\' modifier must follow word or phrase.'); + } + + $lastEntry->processFuzzyProximityModifier($parameter); + + $this->_entries[] = $lastEntry; + } + + /** + * Set boost factor to the entry + * + * @param float $boostFactor + */ + public function boost($boostFactor) + { + // Check, that modifier has came just after word or phrase + if ($this->_nextEntryField !== null || $this->_nextEntrySign !== null) { + throw new Zend_Search_Lucene_Search_QueryParserException('\'^\' modifier must follow word, phrase or subquery.'); + } + + $lastEntry = array_pop($this->_entries); + + if (!$lastEntry instanceof Zend_Search_Lucene_Search_QueryEntry) { + // there are no entries or last entry is boolean operator + throw new Zend_Search_Lucene_Search_QueryParserException('\'^\' modifier must follow word, phrase or subquery.'); + } + + $lastEntry->boost($boostFactor); + + $this->_entries[] = $lastEntry; + } + + /** + * Process logical operator + * + * @param integer $operator + */ + public function addLogicalOperator($operator) + { + if ($this->_mode === self::GM_SIGNS) { + throw new Zend_Search_Lucene_Search_QueryParserException('It\'s not allowed to mix boolean and signs styles in the same subquery.'); + } + + $this->_mode = self::GM_BOOLEAN; + + $this->_entries[] = $operator; + } + + + /** + * Generate 'signs style' query from the context + * '+term1 term2 -term3 +() ...' + * + * @return Zend_Search_Lucene_Search_Query + */ + public function _signStyleExpressionQuery() + { + $query = new Zend_Search_Lucene_Search_Query_Boolean(); + + if (Zend_Search_Lucene_Search_QueryParser::getDefaultOperator() == Zend_Search_Lucene_Search_QueryParser::B_AND) { + $defaultSign = true; // required + } else { + // Zend_Search_Lucene_Search_QueryParser::B_OR + $defaultSign = null; // optional + } + + foreach ($this->_entries as $entryId => $entry) { + $sign = ($this->_signs[$entryId] !== null) ? $this->_signs[$entryId] : $defaultSign; + $query->addSubquery($entry->getQuery($this->_encoding), $sign); + } + + return $query; + } + + + /** + * Generate 'boolean style' query from the context + * 'term1 and term2 or term3 and () and not ()' + * + * @return Zend_Search_Lucene_Search_Query + * @throws Zend_Search_Lucene + */ + private function _booleanExpressionQuery() + { + /** + * We treat each level of an expression as a boolean expression in + * a Disjunctive Normal Form + * + * AND operator has higher precedence than OR + * + * Thus logical query is a disjunction of one or more conjunctions of + * one or more query entries + */ + + $expressionRecognizer = new Zend_Search_Lucene_Search_BooleanExpressionRecognizer(); + + try { + foreach ($this->_entries as $entry) { + if ($entry instanceof Zend_Search_Lucene_Search_QueryEntry) { + $expressionRecognizer->processLiteral($entry); + } else { + switch ($entry) { + case Zend_Search_Lucene_Search_QueryToken::TT_AND_LEXEME: + $expressionRecognizer->processOperator(Zend_Search_Lucene_Search_BooleanExpressionRecognizer::IN_AND_OPERATOR); + break; + + case Zend_Search_Lucene_Search_QueryToken::TT_OR_LEXEME: + $expressionRecognizer->processOperator(Zend_Search_Lucene_Search_BooleanExpressionRecognizer::IN_OR_OPERATOR); + break; + + case Zend_Search_Lucene_Search_QueryToken::TT_NOT_LEXEME: + $expressionRecognizer->processOperator(Zend_Search_Lucene_Search_BooleanExpressionRecognizer::IN_NOT_OPERATOR); + break; + + default: + throw new Zend_Search_Lucene('Boolean expression error. Unknown operator type.'); + } + } + } + + $conjuctions = $expressionRecognizer->finishExpression(); + } catch (Zend_Search_Exception $e) { + // throw new Zend_Search_Lucene_Search_QueryParserException('Boolean expression error. Error message: \'' . + // $e->getMessage() . '\'.' ); + // It's query syntax error message and it should be user friendly. So FSM message is omitted + throw new Zend_Search_Lucene_Search_QueryParserException('Boolean expression error.'); + } + + // Remove 'only negative' conjunctions + foreach ($conjuctions as $conjuctionId => $conjuction) { + $nonNegativeEntryFound = false; + + foreach ($conjuction as $conjuctionEntry) { + if ($conjuctionEntry[1]) { + $nonNegativeEntryFound = true; + break; + } + } + + if (!$nonNegativeEntryFound) { + unset($conjuctions[$conjuctionId]); + } + } + + + $subqueries = array(); + foreach ($conjuctions as $conjuction) { + // Check, if it's a one term conjuction + if (count($conjuction) == 1) { + $subqueries[] = $conjuction[0][0]->getQuery($this->_encoding); + } else { + $subquery = new Zend_Search_Lucene_Search_Query_Boolean(); + + foreach ($conjuction as $conjuctionEntry) { + $subquery->addSubquery($conjuctionEntry[0]->getQuery($this->_encoding), $conjuctionEntry[1]); + } + + $subqueries[] = $subquery; + } + } + + if (count($subqueries) == 0) { + return new Zend_Search_Lucene_Search_Query_Empty(); + } + + if (count($subqueries) == 1) { + return $subqueries[0]; + } + + + $query = new Zend_Search_Lucene_Search_Query_Boolean(); + + foreach ($subqueries as $subquery) { + // Non-requirered entry/subquery + $query->addSubquery($subquery); + } + + return $query; + } + + /** + * Generate query from current context + * + * @return Zend_Search_Lucene_Search_Query + */ + public function getQuery() + { + if ($this->_mode === self::GM_BOOLEAN) { + return $this->_booleanExpressionQuery(); + } else { + return $this->_signStyleExpressionQuery(); + } + } +} diff --git a/search/Zend/Search/Lucene/Search/QueryParserException.php b/search/Zend/Search/Lucene/Search/QueryParserException.php new file mode 100644 index 00000000000..8ca791ffb7e --- /dev/null +++ b/search/Zend/Search/Lucene/Search/QueryParserException.php @@ -0,0 +1,40 @@ +dirroot.'/search/Zend/Search/Lucene/Exception.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Search + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + * + * Special exception type, which may be used to intercept wrong user input + */ +class Zend_Search_Lucene_Search_QueryParserException extends Zend_Search_Lucene_Exception +{} + diff --git a/search/Zend/Search/Lucene/Search/QueryToken.php b/search/Zend/Search/Lucene/Search/QueryToken.php index 56d3522c711..cf153096d2a 100644 --- a/search/Zend/Search/Lucene/Search/QueryToken.php +++ b/search/Zend/Search/Lucene/Search/QueryToken.php @@ -15,46 +15,86 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Exception */ -require_once 'Zend/Search/Lucene/Exception.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_QueryToken { /** - * Token type Word. + * Token types. */ - const TOKTYPE_WORD = 0; + const TT_WORD = 0; // Word + const TT_PHRASE = 1; // Phrase (one or several quoted words) + const TT_FIELD = 2; // Field name in 'field:word', field: or field:() pairs + const TT_FIELD_INDICATOR = 3; // ':' + const TT_REQUIRED = 4; // '+' + const TT_PROHIBITED = 5; // '-' + const TT_FUZZY_PROX_MARK = 6; // '~' + const TT_BOOSTING_MARK = 7; // '^' + const TT_RANGE_INCL_START = 8; // '[' + const TT_RANGE_INCL_END = 9; // ']' + const TT_RANGE_EXCL_START = 10; // '{' + const TT_RANGE_EXCL_END = 11; // '}' + const TT_SUBQUERY_START = 12; // '(' + const TT_SUBQUERY_END = 13; // ')' + const TT_AND_LEXEME = 14; // 'AND' or 'and' + const TT_OR_LEXEME = 15; // 'OR' or 'or' + const TT_NOT_LEXEME = 16; // 'NOT' or 'not' + const TT_TO_LEXEME = 17; // 'TO' or 'to' + const TT_NUMBER = 18; // Number, like: 10, 0.8, .64, .... + /** - * Token type Field. - * Field indicator in 'field:word' pair + * Returns all possible lexeme types. + * It's used for syntax analyzer state machine initialization + * + * @return array */ - const TOKTYPE_FIELD = 1; + public static function getTypes() + { + return array( self::TT_WORD, + self::TT_PHRASE, + self::TT_FIELD, + self::TT_FIELD_INDICATOR, + self::TT_REQUIRED, + self::TT_PROHIBITED, + self::TT_FUZZY_PROX_MARK, + self::TT_BOOSTING_MARK, + self::TT_RANGE_INCL_START, + self::TT_RANGE_INCL_END, + self::TT_RANGE_EXCL_START, + self::TT_RANGE_EXCL_END, + self::TT_SUBQUERY_START, + self::TT_SUBQUERY_END, + self::TT_AND_LEXEME, + self::TT_OR_LEXEME, + self::TT_NOT_LEXEME, + self::TT_TO_LEXEME, + self::TT_NUMBER + ); + } + /** - * Token type Sign. - * '+' (required) or '-' (absentee) sign + * TokenCategories */ - const TOKTYPE_SIGN = 2; - - /** - * Token type Bracket. - * '(' or ')' - */ - const TOKTYPE_BRACKET = 3; + const TC_WORD = 0; // Word + const TC_PHRASE = 1; // Phrase (one or several quoted words) + const TC_NUMBER = 2; // Nubers, which are used with syntax elements. Ex. roam~0.8 + const TC_SYNTAX_ELEMENT = 3; // + - ( ) [ ] { } ! || && ~ ^ /** @@ -71,34 +111,118 @@ class Zend_Search_Lucene_Search_QueryToken */ public $text; + /** + * Token position within query. + * + * @var integer + */ + public $position; + /** * IndexReader constructor needs token type and token text as a parameters. * - * @param $tokType integer - * @param $tokText string + * @param integer $tokenCategory + * @param string $tokText + * @param integer $position */ - public function __construct($tokType, $tokText) + public function __construct($tokenCategory, $tokenText, $position) { - switch ($tokType) { - case self::TOKTYPE_BRACKET: - // fall through to the next case - case self::TOKTYPE_FIELD: - // fall through to the next case - case self::TOKTYPE_SIGN: - // fall through to the next case - case self::TOKTYPE_WORD: + $this->text = $tokenText; + $this->position = $position + 1; // Start from 1 + + switch ($tokenCategory) { + case self::TC_WORD: + if ( strtolower($tokenText) == 'and') { + $this->type = self::TT_AND_LEXEME; + } else if (strtolower($tokenText) == 'or') { + $this->type = self::TT_OR_LEXEME; + } else if (strtolower($tokenText) == 'not') { + $this->type = self::TT_NOT_LEXEME; + } else if (strtolower($tokenText) == 'to') { + $this->type = self::TT_TO_LEXEME; + } else { + $this->type = self::TT_WORD; + } break; + + case self::TC_PHRASE: + $this->type = self::TT_PHRASE; + break; + + case self::TC_NUMBER: + $this->type = self::TT_NUMBER; + break; + + case self::TC_SYNTAX_ELEMENT: + switch ($tokenText) { + case ':': + $this->type = self::TT_FIELD_INDICATOR; + break; + + case '+': + $this->type = self::TT_REQUIRED; + break; + + case '-': + $this->type = self::TT_PROHIBITED; + break; + + case '~': + $this->type = self::TT_FUZZY_PROX_MARK; + break; + + case '^': + $this->type = self::TT_BOOSTING_MARK; + break; + + case '[': + $this->type = self::TT_RANGE_INCL_START; + break; + + case ']': + $this->type = self::TT_RANGE_INCL_END; + break; + + case '{': + $this->type = self::TT_RANGE_EXCL_START; + break; + + case '}': + $this->type = self::TT_RANGE_EXCL_END; + break; + + case '(': + $this->type = self::TT_SUBQUERY_START; + break; + + case ')': + $this->type = self::TT_SUBQUERY_END; + break; + + case '!': + $this->type = self::TT_NOT_LEXEME; + break; + + case '&&': + $this->type = self::TT_AND_LEXEME; + break; + + case '||': + $this->type = self::TT_OR_LEXEME; + break; + + default: + throw new Zend_Search_Lucene_Exception('Unrecognized query syntax lexeme: \'' . $tokenText . '\''); + } + break; + + case self::TC_NUMBER: + $this->type = self::TT_NUMBER; + default: - throw new Zend_Search_Lucene_Exception("Unrecognized token type \"$tokType\"."); + throw new Zend_Search_Lucene_Exception('Unrecognized lexeme type: \'' . $tokenCategory . '\''); } - - if (!strlen($tokText)) { - throw new Zend_Search_Lucene_Exception('Token text must be supplied.'); - } - - $this->type = $tokType; - $this->text = $tokText; } } diff --git a/search/Zend/Search/Lucene/Search/QueryTokenizer.php b/search/Zend/Search/Lucene/Search/QueryTokenizer.php index 4fe870bedc1..2965eaa7538 100644 --- a/search/Zend/Search/Lucene/Search/QueryTokenizer.php +++ b/search/Zend/Search/Lucene/Search/QueryTokenizer.php @@ -21,10 +21,10 @@ /** Zend_Search_Lucene_Search_QueryToken */ -require_once 'Zend/Search/Lucene/Search/QueryToken.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryToken.php'; /** Zend_Search_Lucene_Exception */ -require_once 'Zend/Search/Lucene/Exception.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; /** diff --git a/search/Zend/Search/Lucene/Search/Similarity.php b/search/Zend/Search/Lucene/Search/Similarity.php index 74ecb1dda67..016d232a244 100644 --- a/search/Zend/Search/Lucene/Search/Similarity.php +++ b/search/Zend/Search/Lucene/Search/Similarity.php @@ -15,20 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Similarity_Default */ -require_once 'Zend/Search/Lucene/Search/Similarity/Default.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Similarity/Default.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_Search_Similarity @@ -38,7 +38,7 @@ abstract class Zend_Search_Lucene_Search_Similarity * * @var Zend_Search_Lucene_Search_Similarity */ - static private $_defaultImpl; + private static $_defaultImpl; /** * Cache of decoded bytes. @@ -46,7 +46,7 @@ abstract class Zend_Search_Lucene_Search_Similarity * * @var array */ - static private $_normTable = array( 0 => 0.0, + private static $_normTable = array( 0 => 0.0, 1 => 5.820766E-10, 2 => 6.9849193E-10, 3 => 8.1490725E-10, @@ -310,7 +310,7 @@ abstract class Zend_Search_Lucene_Search_Similarity * * @param Zend_Search_Lucene_Search_Similarity $similarity */ - static public function setDefault(Zend_Search_Lucene_Search_Similarity $similarity) + public static function setDefault(Zend_Search_Lucene_Search_Similarity $similarity) { self::$_defaultImpl = $similarity; } @@ -322,7 +322,7 @@ abstract class Zend_Search_Lucene_Search_Similarity * * @return Zend_Search_Lucene_Search_Similarity */ - static public function getDefault() + public static function getDefault() { if (!self::$_defaultImpl instanceof Zend_Search_Lucene_Search_Similarity) { self::$_defaultImpl = new Zend_Search_Lucene_Search_Similarity_Default(); @@ -381,7 +381,7 @@ abstract class Zend_Search_Lucene_Search_Similarity * @param integer $byte * @return float */ - static public function decodeNorm($byte) + public static function decodeNorm($byte) { return self::$_normTable[$byte & 0xFF]; } @@ -412,7 +412,7 @@ abstract class Zend_Search_Lucene_Search_Similarity * @param integer $b * @return float */ - static private function _floatToByte($f) + private static function _floatToByte($f) { // round negatives up to zero if ($f <= 0.0) { @@ -495,10 +495,10 @@ abstract class Zend_Search_Lucene_Search_Similarity * Returns a score factor for the term * * @param mixed $input - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader * @return a score factor for the term */ - public function idf($input, $reader) + public function idf($input, Zend_Search_Lucene_Interface $reader) { if (!is_array($input)) { return $this->idfFreq($reader->docFreq($input), $reader->count()); diff --git a/search/Zend/Search/Lucene/Search/Similarity/Default.php b/search/Zend/Search/Lucene/Search/Similarity/Default.php index 6cafb59668d..8263f2a72f8 100644 --- a/search/Zend/Search/Lucene/Search/Similarity/Default.php +++ b/search/Zend/Search/Lucene/Search/Similarity/Default.php @@ -15,16 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ +/** Zend_Search_Lucene_Search_Similarity */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Similarity.php'; + + /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Similarity_Default extends Zend_Search_Lucene_Search_Similarity diff --git a/search/Zend/Search/Lucene/Search/Weight.php b/search/Zend/Search/Lucene/Search/Weight.php index 248f5cb2b41..2faba2861bf 100644 --- a/search/Zend/Search/Lucene/Search/Weight.php +++ b/search/Zend/Search/Lucene/Search/Weight.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -32,17 +32,40 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_Search_Weight { + /** + * Normalization factor. + * This value is stored only for query expanation purpose and not used in any other place + * + * @var float + */ + protected $_queryNorm; + + /** + * Weight value + * + * Weight value may be initialized in sumOfSquaredWeights() or normalize() + * because they both are invoked either in Query::_initWeight (for top-level query) or + * in corresponding methods of parent query's weights + * + * @var float + */ + protected $_value; + + /** * The weight for this query. * * @return float */ - abstract public function getValue(); + public function getValue() + { + return $this->_value; + } /** * The sum of squared weights of contained query clauses. diff --git a/search/Zend/Search/Lucene/Search/Weight/Boolean.php b/search/Zend/Search/Lucene/Search/Weight/Boolean.php new file mode 100644 index 00000000000..7a42ed28269 --- /dev/null +++ b/search/Zend/Search/Lucene/Search/Weight/Boolean.php @@ -0,0 +1,136 @@ +dirroot.'/search/Zend/Search/Lucene/Search/Weight.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Search + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Search_Weight_Boolean extends Zend_Search_Lucene_Search_Weight +{ + /** + * IndexReader. + * + * @var Zend_Search_Lucene_Interface + */ + private $_reader; + + /** + * The query that this concerns. + * + * @var Zend_Search_Lucene_Search_Query + */ + private $_query; + + /** + * Queries weights + * Array of Zend_Search_Lucene_Search_Weight + * + * @var array + */ + private $_weights; + + + /** + * Zend_Search_Lucene_Search_Weight_Boolean constructor + * query - the query that this concerns. + * reader - index reader + * + * @param Zend_Search_Lucene_Search_Query $query + * @param Zend_Search_Lucene_Interface $reader + */ + public function __construct(Zend_Search_Lucene_Search_Query $query, + Zend_Search_Lucene_Interface $reader) + { + $this->_query = $query; + $this->_reader = $reader; + $this->_weights = array(); + + $signs = $query->getSigns(); + + foreach ($query->getSubqueries() as $num => $subquery) { + if ($signs === null || $signs[$num] === null || $signs[$num]) { + $this->_weights[$num] = $subquery->createWeight($reader); + } + } + } + + + /** + * The weight for this query + * Standard Weight::$_value is not used for boolean queries + * + * @return float + */ + public function getValue() + { + return $this->_query->getBoost(); + } + + + /** + * The sum of squared weights of contained query clauses. + * + * @return float + */ + public function sumOfSquaredWeights() + { + $sum = 0; + foreach ($this->_weights as $weight) { + // sum sub weights + $sum += $weight->sumOfSquaredWeights(); + } + + // boost each sub-weight + $sum *= $this->_query->getBoost() * $this->_query->getBoost(); + + // check for empty query (like '-something -another') + if ($sum == 0) { + $sum = 1.0; + } + return $sum; + } + + + /** + * Assigns the query normalization factor to this. + * + * @param float $queryNorm + */ + public function normalize($queryNorm) + { + // incorporate boost + $queryNorm *= $this->_query->getBoost(); + + foreach ($this->_weights as $weight) { + $weight->normalize($queryNorm); + } + } +} + + diff --git a/search/Zend/Search/Lucene/Search/Weight/Empty.php b/search/Zend/Search/Lucene/Search/Weight/Empty.php new file mode 100644 index 00000000000..00c1bce4517 --- /dev/null +++ b/search/Zend/Search/Lucene/Search/Weight/Empty.php @@ -0,0 +1,56 @@ +dirroot.'/search/Zend/Search/Lucene/Search/Weight.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Search + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Search_Weight_Empty extends Zend_Search_Lucene_Search_Weight +{ + /** + * The sum of squared weights of contained query clauses. + * + * @return float + */ + public function sumOfSquaredWeights() + { + return 1; + } + + + /** + * Assigns the query normalization factor to this. + * + * @param float $queryNorm + */ + public function normalize($queryNorm) + { + } +} + diff --git a/search/Zend/Search/Lucene/Search/Weight/MultiTerm.php b/search/Zend/Search/Lucene/Search/Weight/MultiTerm.php index 448bb064ebc..122cfa4035a 100644 --- a/search/Zend/Search/Lucene/Search/Weight/MultiTerm.php +++ b/search/Zend/Search/Lucene/Search/Weight/MultiTerm.php @@ -15,20 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Weight */ -require_once 'Zend/Search/Lucene/Search/Weight.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Weight_MultiTerm extends Zend_Search_Lucene_Search_Weight @@ -36,14 +36,14 @@ class Zend_Search_Lucene_Search_Weight_MultiTerm extends Zend_Search_Lucene_Sear /** * IndexReader. * - * @var Zend_Search_Lucene + * @var Zend_Search_Lucene_Interface */ private $_reader; /** * The query that this concerns. * - * @var Zend_Search_Lucene_Search_Query_MultiTerm + * @var Zend_Search_Lucene_Search_Query */ private $_query; @@ -61,10 +61,11 @@ class Zend_Search_Lucene_Search_Weight_MultiTerm extends Zend_Search_Lucene_Sear * query - the query that this concerns. * reader - index reader * - * @param Zend_Search_Lucene_Search_Query_MultiTerm $query - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Search_Query $query + * @param Zend_Search_Lucene_Interface $reader */ - public function __construct($query, $reader) + public function __construct(Zend_Search_Lucene_Search_Query $query, + Zend_Search_Lucene_Interface $reader) { $this->_query = $query; $this->_reader = $reader; @@ -72,10 +73,10 @@ class Zend_Search_Lucene_Search_Weight_MultiTerm extends Zend_Search_Lucene_Sear $signs = $query->getSigns(); - foreach ($query->getTerms() as $num => $term) { - if ($signs === null || $signs[$num] === null || $signs[$num]) { - $this->_weights[$num] = new Zend_Search_Lucene_Search_Weight_Term($term, $query, $reader); - $query->setWeight($num, $this->_weights[$num]); + foreach ($query->getTerms() as $id => $term) { + if ($signs === null || $signs[$id] === null || $signs[$id]) { + $this->_weights[$id] = new Zend_Search_Lucene_Search_Weight_Term($term, $query, $reader); + $query->setWeight($id, $this->_weights[$id]); } } } @@ -83,6 +84,7 @@ class Zend_Search_Lucene_Search_Weight_MultiTerm extends Zend_Search_Lucene_Sear /** * The weight for this query + * Standard Weight::$_value is not used for boolean queries * * @return float */ diff --git a/search/Zend/Search/Lucene/Search/Weight/Phrase.php b/search/Zend/Search/Lucene/Search/Weight/Phrase.php index 536659614cb..7faaa7ad130 100644 --- a/search/Zend/Search/Lucene/Search/Weight/Phrase.php +++ b/search/Zend/Search/Lucene/Search/Weight/Phrase.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -23,14 +23,14 @@ /** * Zend_Search_Lucene_Search_Weight */ -require_once 'Zend/Search/Lucene/Search/Weight.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Weight_Phrase extends Zend_Search_Lucene_Search_Weight @@ -38,7 +38,7 @@ class Zend_Search_Lucene_Search_Weight_Phrase extends Zend_Search_Lucene_Search_ /** * IndexReader. * - * @var Zend_Search_Lucene + * @var Zend_Search_Lucene_Interface */ private $_reader; @@ -49,13 +49,6 @@ class Zend_Search_Lucene_Search_Weight_Phrase extends Zend_Search_Lucene_Search_ */ private $_query; - /** - * Weight value - * - * @var float - */ - private $_value; - /** * Score factor * @@ -63,46 +56,19 @@ class Zend_Search_Lucene_Search_Weight_Phrase extends Zend_Search_Lucene_Search_ */ private $_idf; - /** - * Normalization factor - * - * @var float - */ - private $_queryNorm; - - - /** - * Query weight - * - * @var float - */ - private $_queryWeight; - - /** * Zend_Search_Lucene_Search_Weight_Phrase constructor * * @param Zend_Search_Lucene_Search_Query_Phrase $query - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Interface $reader */ - public function __construct(Zend_Search_Lucene_Search_Query_Phrase $query, Zend_Search_Lucene $reader) + public function __construct(Zend_Search_Lucene_Search_Query_Phrase $query, + Zend_Search_Lucene_Interface $reader) { $this->_query = $query; $this->_reader = $reader; } - - /** - * The weight for this query - * - * @return float - */ - public function getValue() - { - return $this->_value; - } - - /** * The sum of squared weights of contained query clauses. * diff --git a/search/Zend/Search/Lucene/Search/Weight/Term.php b/search/Zend/Search/Lucene/Search/Weight/Term.php index d502896a5b7..478cb2a8058 100644 --- a/search/Zend/Search/Lucene/Search/Weight/Term.php +++ b/search/Zend/Search/Lucene/Search/Weight/Term.php @@ -15,20 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Weight */ -require_once 'Zend/Search/Lucene/Search/Weight.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Weight_Term extends Zend_Search_Lucene_Search_Weight @@ -36,7 +36,7 @@ class Zend_Search_Lucene_Search_Weight_Term extends Zend_Search_Lucene_Search_We /** * IndexReader. * - * @var Zend_Search_Lucene + * @var Zend_Search_Lucene_Interface */ private $_reader; @@ -54,13 +54,6 @@ class Zend_Search_Lucene_Search_Weight_Term extends Zend_Search_Lucene_Search_We */ private $_query; - /** - * Weight value - * - * @var float - */ - private $_value; - /** * Score factor * @@ -68,14 +61,6 @@ class Zend_Search_Lucene_Search_Weight_Term extends Zend_Search_Lucene_Search_We */ private $_idf; - /** - * Normalization factor - * - * @var float - */ - private $_queryNorm; - - /** * Query weight * @@ -88,9 +73,13 @@ class Zend_Search_Lucene_Search_Weight_Term extends Zend_Search_Lucene_Search_We * Zend_Search_Lucene_Search_Weight_Term constructor * reader - index reader * - * @param Zend_Search_Lucene $reader + * @param Zend_Search_Lucene_Index_Term $term + * @param Zend_Search_Lucene_Search_Query $query + * @param Zend_Search_Lucene_Interface $reader */ - public function __construct($term, $query, $reader) + public function __construct(Zend_Search_Lucene_Index_Term $term, + Zend_Search_Lucene_Search_Query $query, + Zend_Search_Lucene_Interface $reader) { $this->_term = $term; $this->_query = $query; @@ -98,17 +87,6 @@ class Zend_Search_Lucene_Search_Weight_Term extends Zend_Search_Lucene_Search_We } - /** - * The weight for this query - * - * @return float - */ - public function getValue() - { - return $this->_value; - } - - /** * The sum of squared weights of contained query clauses. * diff --git a/search/Zend/Search/Lucene/Storage/Directory.php b/search/Zend/Search/Lucene/Storage/Directory.php index 01ea380e3c8..67e2b295f52 100644 --- a/search/Zend/Search/Lucene/Storage/Directory.php +++ b/search/Zend/Search/Lucene/Storage/Directory.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -24,7 +24,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_Storage_Directory @@ -111,10 +111,16 @@ abstract class Zend_Search_Lucene_Storage_Directory /** * Returns a Zend_Search_Lucene_Storage_File object for a given $filename in the directory. * + * If $shareHandler option is true, then file handler can be shared between File Object + * requests. It speed-ups performance, but makes problems with file position. + * Shared handler are good for short atomic requests. + * Non-shared handlers are useful for stream file reading (especial for compound files). + * * @param string $filename + * @param boolean $shareHandler * @return Zend_Search_Lucene_Storage_File */ - abstract public function getFileObject($filename); + abstract public function getFileObject($filename, $shareHandler = true); } diff --git a/search/Zend/Search/Lucene/Storage/Directory/Filesystem.php b/search/Zend/Search/Lucene/Storage/Directory/Filesystem.php index 8d675c35b5d..7ac76810ab6 100644 --- a/search/Zend/Search/Lucene/Storage/Directory/Filesystem.php +++ b/search/Zend/Search/Lucene/Storage/Directory/Filesystem.php @@ -15,16 +15,16 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Storage_Directory */ -require_once 'Zend/Search/Lucene/Storage/Directory.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/Directory.php'; /** Zend_Search_Lucene_Storage_File_Filesystem */ -require_once 'Zend/Search/Lucene/Storage/File/Filesystem.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/File/Filesystem.php'; /** @@ -33,7 +33,7 @@ require_once 'Zend/Search/Lucene/Storage/File/Filesystem.php'; * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene_Storage_Directory @@ -64,7 +64,7 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene * @return boolean */ - static public function mkdirs($dir, $mode = 0777, $recursive = true) + public static function mkdirs($dir, $mode = 0777, $recursive = true) { if (is_null($dir) || $dir === '') { return false; @@ -113,7 +113,7 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene $fileObject->close(); } - unset($this->_fileHandlers); + $this->_fileHandlers = array(); } @@ -127,15 +127,14 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene $result = array(); $dirContent = opendir( $this->_dirPath ); - while ($file = readdir($dirContent)) { + while (($file = readdir($dirContent)) !== false) { if (($file == '..')||($file == '.')) continue; - $fullName = $this->_dirPath . '/' . $file; - if( !is_dir($this->_dirPath . '/' . $file) ) { $result[] = $file; } } + closedir($dirContent); return $result; } @@ -165,11 +164,17 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene */ public function deleteFile($filename) { + /** + * @todo add support of "deletable" file + * "deletable" is used on Windows systems if file can't be deleted + * (while it is still open). + */ + if (isset($this->_fileHandlers[$filename])) { $this->_fileHandlers[$filename]->close(); } unset($this->_fileHandlers[$filename]); - unlink($this->_dirPath .'/'. $filename); + unlink($this->_dirPath . '/' . $filename); } @@ -219,24 +224,40 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene * @param string $from * @param string $to * @return void + * @throws Zend_Search_Lucene_Exception */ public function renameFile($from, $to) { - if ($this->_fileHandlers[$from] !== null) { + global $php_errormsg; + + if (isset($this->_fileHandlers[$from])) { $this->_fileHandlers[$from]->close(); } unset($this->_fileHandlers[$from]); - if ($this->_fileHandlers[$to] !== null) { + if (isset($this->_fileHandlers[$to])) { $this->_fileHandlers[$to]->close(); } unset($this->_fileHandlers[$to]); if (file_exists($this->_dirPath . '/' . $to)) { - unlink($this->_dirPath . '/' . $to); + if (!unlink($this->_dirPath . '/' . $to)) { + throw new Zend_Search_Lucene_Exception('Delete operation failed'); + } } - return @rename($this->_dirPath . '/' . $from, $this->_dirPath . '/' . $to); + $trackErrors = ini_get('track_errors'); + ini_set('track_errors', '1'); + + $success = @rename($this->_dirPath . '/' . $from, $this->_dirPath . '/' . $to); + if (!$success) { + ini_set('track_errors', $trackErrors); + throw new Zend_Search_Lucene_Exception($php_errormsg); + } + + ini_set('track_errors', $trackErrors); + + return $success; } @@ -255,17 +276,29 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene /** * Returns a Zend_Search_Lucene_Storage_File object for a given $filename in the directory. * + * If $shareHandler option is true, then file handler can be shared between File Object + * requests. It speed-ups performance, but makes problems with file position. + * Shared handler are good for short atomic requests. + * Non-shared handlers are useful for stream file reading (especial for compound files). + * * @param string $filename + * @param boolean $shareHandler * @return Zend_Search_Lucene_Storage_File */ - public function getFileObject($filename) + public function getFileObject($filename, $shareHandler = true) { + $fullFilename = $this->_dirPath . '/' . $filename; + + if (!$shareHandler) { + return new Zend_Search_Lucene_Storage_File_Filesystem($fullFilename); + } + if (isset( $this->_fileHandlers[$filename] )) { $this->_fileHandlers[$filename]->seek(0); return $this->_fileHandlers[$filename]; } - $this->_fileHandlers[$filename] = new Zend_Search_Lucene_Storage_File_Filesystem($this->_dirPath . '/' . $filename); + $this->_fileHandlers[$filename] = new Zend_Search_Lucene_Storage_File_Filesystem($fullFilename); return $this->_fileHandlers[$filename]; } } diff --git a/search/Zend/Search/Lucene/Storage/File.php b/search/Zend/Search/Lucene/Storage/File.php index 5a195ae85f4..8370c719db7 100644 --- a/search/Zend/Search/Lucene/Storage/File.php +++ b/search/Zend/Search/Lucene/Storage/File.php @@ -15,21 +15,21 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Exception */ -require_once 'Zend/Search/Lucene/Exception.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_Storage_File @@ -69,6 +69,15 @@ abstract class Zend_Search_Lucene_Storage_File */ abstract public function tell(); + /** + * Flush output. + * + * Returns true on success or false on failure. + * + * @return boolean + */ + abstract public function flush(); + /** * Writes $length number of bytes (all, if $length===null) to the end * of the file. @@ -78,6 +87,20 @@ abstract class Zend_Search_Lucene_Storage_File */ abstract protected function _fwrite($data, $length=null); + /** + * Lock file + * + * Lock type may be a LOCK_SH (shared lock) or a LOCK_EX (exclusive lock) + * + * @param integer $lockType + * @return boolean + */ + abstract public function lock($lockType, $nonBlockinLock = false); + + /** + * Unlock file + */ + abstract public function unlock(); /** * Reads a byte from the current position in the file @@ -401,4 +424,4 @@ abstract class Zend_Search_Lucene_Storage_File { return $this->_fread($this->readVInt()); } -} \ No newline at end of file +} diff --git a/search/Zend/Search/Lucene/Storage/File/Filesystem.php b/search/Zend/Search/Lucene/Storage/File/Filesystem.php index 7c33543dd6e..1f2097eb4f6 100644 --- a/search/Zend/Search/Lucene/Storage/File/Filesystem.php +++ b/search/Zend/Search/Lucene/Storage/File/Filesystem.php @@ -15,23 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Storage_File */ -require_once 'Zend/Search/Lucene/Storage/File.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/File.php'; /** Zend_Search_Lucene_Exception */ -require_once 'Zend/Search/Lucene/Exception.php'; +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Storage_File_Filesystem extends Zend_Search_Lucene_Storage_File @@ -53,12 +53,12 @@ class Zend_Search_Lucene_Storage_File_Filesystem extends Zend_Search_Lucene_Stor { global $php_errormsg; - $trackErrors = ini_get( "track_errors"); + $trackErrors = ini_get('track_errors'); ini_set('track_errors', '1'); $this->_fileHandle = @fopen($filename, $mode); - if ($this->_fileHandle===false) { + if ($this->_fileHandle === false) { ini_set('track_errors', $trackErrors); throw new Zend_Search_Lucene_Exception($php_errormsg); } @@ -100,6 +100,17 @@ class Zend_Search_Lucene_Storage_File_Filesystem extends Zend_Search_Lucene_Stor return ftell($this->_fileHandle); } + /** + * Flush output. + * + * Returns true on success or false on failure. + * + * @return boolean + */ + public function flush() + { + return fflush($this->_fileHandle); + } /** * Close File object @@ -167,5 +178,39 @@ class Zend_Search_Lucene_Storage_File_Filesystem extends Zend_Search_Lucene_Stor fwrite($this->_fileHandle, $data, $length); } } + + /** + * Lock file + * + * Lock type may be a LOCK_SH (shared lock) or a LOCK_EX (exclusive lock) + * + * @param integer $lockType + * @param boolean $nonBlockinLock + * @return boolean + */ + public function lock($lockType, $nonBlockinLock = false) + { + if ($nonBlockinLock) { + return flock($this->_fileHandle, $lockType | LOCK_NB); + } else { + return flock($this->_fileHandle, $lockType); + } + } + + /** + * Unlock file + * + * Returns true on success + * + * @return boolean + */ + public function unlock() + { + if ($this->_fileHandle !== null ) { + return flock($this->_fileHandle, LOCK_UN); + } else { + return true; + } + } } diff --git a/search/Zend/Search/Lucene/Storage/File/Memory.php b/search/Zend/Search/Lucene/Storage/File/Memory.php new file mode 100644 index 00000000000..e3567c66dab --- /dev/null +++ b/search/Zend/Search/Lucene/Storage/File/Memory.php @@ -0,0 +1,555 @@ +dirroot.'/search/Zend/Search/Lucene/Storage/File.php'; + +/** Zend_Search_Lucene_Exception */ +require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; + + +/** + * @category Zend + * @package Zend_Search_Lucene + * @subpackage Storage + * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @license http://framework.zend.com/license/new-bsd New BSD License + */ +class Zend_Search_Lucene_Storage_File_Memory extends Zend_Search_Lucene_Storage_File +{ + /** + * FileData + * + * @var string + */ + private $_data; + + /** + * File Position + * + * @var integer + */ + private $_position = 0; + + + /** + * Object constractor + * + * @param string $data + */ + public function __construct($data) + { + $this->_data = $data; + } + + /** + * Reads $length number of bytes at the current position in the + * file and advances the file pointer. + * + * @param integer $length + * @return string + */ + protected function _fread($length = 1) + { + $returnValue = substr($this->_data, $this->_position, $length); + $this->_position += $length; + return $returnValue; + } + + + /** + * Sets the file position indicator and advances the file pointer. + * The new position, measured in bytes from the beginning of the file, + * is obtained by adding offset to the position specified by whence, + * whose values are defined as follows: + * SEEK_SET - Set position equal to offset bytes. + * SEEK_CUR - Set position to current location plus offset. + * SEEK_END - Set position to end-of-file plus offset. (To move to + * a position before the end-of-file, you need to pass a negative value + * in offset.) + * Upon success, returns 0; otherwise, returns -1 + * + * @param integer $offset + * @param integer $whence + * @return integer + */ + public function seek($offset, $whence=SEEK_SET) + { + switch ($whence) { + case SEEK_SET: + $this->_position = $offset; + break; + + case SEEK_CUR: + $this->_position += $offset; + break; + + case SEEK_END: + $this->_position = strlen($this->_data); + $this->_position += $offset; + break; + + default: + break; + } + } + + /** + * Get file position. + * + * @return integer + */ + public function tell() + { + return $this->_position; + } + + /** + * Flush output. + * + * Returns true on success or false on failure. + * + * @return boolean + */ + public function flush() + { + // Do nothing + + return true; + } + + /** + * Writes $length number of bytes (all, if $length===null) to the end + * of the file. + * + * @param string $data + * @param integer $length + */ + protected function _fwrite($data, $length=null) + { + // We do not need to check if file position points to the end of "file". + // Only append operation is supported now + + if ($length !== null) { + $this->_data .= substr($data, 0, $length); + } else { + $this->_data .= $data; + } + + $this->_position = strlen($this->_data); + } + + /** + * Lock file + * + * Lock type may be a LOCK_SH (shared lock) or a LOCK_EX (exclusive lock) + * + * @param integer $lockType + * @return boolean + */ + public function lock($lockType, $nonBlockinLock = false) + { + // Memory files can't be shared + // do nothing + + return true; + } + + /** + * Unlock file + */ + public function unlock() + { + // Memory files can't be shared + // do nothing + } + + /** + * Reads a byte from the current position in the file + * and advances the file pointer. + * + * @return integer + */ + public function readByte() + { + return ord($this->_data[$this->_position++]); + } + + /** + * Writes a byte to the end of the file. + * + * @param integer $byte + */ + public function writeByte($byte) + { + // We do not need to check if file position points to the end of "file". + // Only append operation is supported now + + $this->_data .= chr($byte); + $this->_position = strlen($this->_data); + + return 1; + } + + /** + * Read num bytes from the current position in the file + * and advances the file pointer. + * + * @param integer $num + * @return string + */ + public function readBytes($num) + { + $returnValue = substr($this->_data, $this->_position, $num); + $this->_position += $num; + + return $returnValue; + } + + /** + * Writes num bytes of data (all, if $num===null) to the end + * of the string. + * + * @param string $data + * @param integer $num + */ + public function writeBytes($data, $num=null) + { + // We do not need to check if file position points to the end of "file". + // Only append operation is supported now + + if ($num !== null) { + $this->_data .= substr($data, 0, $num); + } else { + $this->_data .= $data; + } + + $this->_position = strlen($this->_data); + } + + + /** + * Reads an integer from the current position in the file + * and advances the file pointer. + * + * @return integer + */ + public function readInt() + { + $str = substr($this->_data, $this->_position, 4); + $this->_position += 4; + + return ord($str{0}) << 24 | + ord($str{1}) << 16 | + ord($str{2}) << 8 | + ord($str{3}); + } + + + /** + * Writes an integer to the end of file. + * + * @param integer $value + */ + public function writeInt($value) + { + // We do not need to check if file position points to the end of "file". + // Only append operation is supported now + + settype($value, 'integer'); + $this->_data .= chr($value>>24 & 0xFF) . + chr($value>>16 & 0xFF) . + chr($value>>8 & 0xFF) . + chr($value & 0xFF); + + $this->_position = strlen($this->_data); + } + + + /** + * Returns a long integer from the current position in the file + * and advances the file pointer. + * + * @return integer + * @throws Zend_Search_Lucene_Exception + */ + public function readLong() + { + $str = substr($this->_data, $this->_position, 8); + $this->_position += 8; + + /** + * Check, that we work in 64-bit mode. + * fseek() uses long for offset. Thus, largest index segment file size in 32bit mode is 2Gb + */ + if (PHP_INT_SIZE > 4) { + return ord($str{0}) << 56 | + ord($str{1}) << 48 | + ord($str{2}) << 40 | + ord($str{3}) << 32 | + ord($str{4}) << 24 | + ord($str{5}) << 16 | + ord($str{6}) << 8 | + ord($str{7}); + } else { + if ((ord($str{0}) != 0) || + (ord($str{1}) != 0) || + (ord($str{2}) != 0) || + (ord($str{3}) != 0) || + ((ord($str{0}) & 0x80) != 0)) { + throw new Zend_Search_Lucene_Exception('Largest supported segment size (for 32-bit mode) is 2Gb'); + } + + return ord($str{4}) << 24 | + ord($str{5}) << 16 | + ord($str{6}) << 8 | + ord($str{7}); + } + } + + /** + * Writes long integer to the end of file + * + * @param integer $value + * @throws Zend_Search_Lucene_Exception + */ + public function writeLong($value) + { + // We do not need to check if file position points to the end of "file". + // Only append operation is supported now + + /** + * Check, that we work in 64-bit mode. + * fseek() and ftell() use long for offset. Thus, largest index segment file size in 32bit mode is 2Gb + */ + if (PHP_INT_SIZE > 4) { + settype($value, 'integer'); + $this->_data .= chr($value>>56 & 0xFF) . + chr($value>>48 & 0xFF) . + chr($value>>40 & 0xFF) . + chr($value>>32 & 0xFF) . + chr($value>>24 & 0xFF) . + chr($value>>16 & 0xFF) . + chr($value>>8 & 0xFF) . + chr($value & 0xFF); + } else { + if ($value > 0x7FFFFFFF) { + throw new Zend_Search_Lucene_Exception('Largest supported segment size (for 32-bit mode) is 2Gb'); + } + + $this->_data .= chr(0) . chr(0) . chr(0) . chr(0) . + chr($value>>24 & 0xFF) . + chr($value>>16 & 0xFF) . + chr($value>>8 & 0xFF) . + chr($value & 0xFF); + } + + $this->_position = strlen($this->_data); + } + + + + /** + * Returns a variable-length integer from the current + * position in the file and advances the file pointer. + * + * @return integer + */ + public function readVInt() + { + $nextByte = ord($this->_data[$this->_position++]); + $val = $nextByte & 0x7F; + + for ($shift=7; ($nextByte & 0x80) != 0; $shift += 7) { + $nextByte = ord($this->_data[$this->_position++]); + $val |= ($nextByte & 0x7F) << $shift; + } + return $val; + } + + /** + * Writes a variable-length integer to the end of file. + * + * @param integer $value + */ + public function writeVInt($value) + { + // We do not need to check if file position points to the end of "file". + // Only append operation is supported now + + settype($value, 'integer'); + while ($value > 0x7F) { + $this->_data .= chr( ($value & 0x7F)|0x80 ); + $value >>= 7; + } + $this->_data .= chr($value); + + $this->_position = strlen($this->_data); + } + + + /** + * Reads a string from the current position in the file + * and advances the file pointer. + * + * @return string + */ + public function readString() + { + $strlen = $this->readVInt(); + if ($strlen == 0) { + return ''; + } else { + /** + * This implementation supports only Basic Multilingual Plane + * (BMP) characters (from 0x0000 to 0xFFFF) and doesn't support + * "supplementary characters" (characters whose code points are + * greater than 0xFFFF) + * Java 2 represents these characters as a pair of char (16-bit) + * values, the first from the high-surrogates range (0xD800-0xDBFF), + * the second from the low-surrogates range (0xDC00-0xDFFF). Then + * they are encoded as usual UTF-8 characters in six bytes. + * Standard UTF-8 representation uses four bytes for supplementary + * characters. + */ + + $str_val = substr($this->_data, $this->_position, $strlen); + $this->_position += $strlen; + + for ($count = 0; $count < $strlen; $count++ ) { + if (( ord($str_val{$count}) & 0xC0 ) == 0xC0) { + $addBytes = 1; + if (ord($str_val{$count}) & 0x20 ) { + $addBytes++; + + // Never used. Java2 doesn't encode strings in four bytes + if (ord($str_val{$count}) & 0x10 ) { + $addBytes++; + } + } + $str_val .= substr($this->_data, $this->_position, $addBytes); + $this->_position += $addBytes; + $strlen += $addBytes; + + // Check for null character. Java2 encodes null character + // in two bytes. + if (ord($str_val{$count}) == 0xC0 && + ord($str_val{$count+1}) == 0x80 ) { + $str_val{$count} = 0; + $str_val = substr($str_val,0,$count+1) + . substr($str_val,$count+2); + } + $count += $addBytes; + } + } + + return $str_val; + } + } + + /** + * Writes a string to the end of file. + * + * @param string $str + * @throws Zend_Search_Lucene_Exception + */ + public function writeString($str) + { + /** + * This implementation supports only Basic Multilingual Plane + * (BMP) characters (from 0x0000 to 0xFFFF) and doesn't support + * "supplementary characters" (characters whose code points are + * greater than 0xFFFF) + * Java 2 represents these characters as a pair of char (16-bit) + * values, the first from the high-surrogates range (0xD800-0xDBFF), + * the second from the low-surrogates range (0xDC00-0xDFFF). Then + * they are encoded as usual UTF-8 characters in six bytes. + * Standard UTF-8 representation uses four bytes for supplementary + * characters. + */ + + // We do not need to check if file position points to the end of "file". + // Only append operation is supported now + + // convert input to a string before iterating string characters + settype($str, 'string'); + + $chars = $strlen = strlen($str); + $containNullChars = false; + + for ($count = 0; $count < $strlen; $count++ ) { + /** + * String is already in Java 2 representation. + * We should only calculate actual string length and replace + * \x00 by \xC0\x80 + */ + if ((ord($str{$count}) & 0xC0) == 0xC0) { + $addBytes = 1; + if (ord($str{$count}) & 0x20 ) { + $addBytes++; + + // Never used. Java2 doesn't encode strings in four bytes + // and we dont't support non-BMP characters + if (ord($str{$count}) & 0x10 ) { + $addBytes++; + } + } + $chars -= $addBytes; + + if (ord($str{$count}) == 0 ) { + $containNullChars = true; + } + $count += $addBytes; + } + } + + if ($chars < 0) { + throw new Zend_Search_Lucene_Exception('Invalid UTF-8 string'); + } + + $this->writeVInt($chars); + if ($containNullChars) { + $this->_data .= str_replace($str, "\x00", "\xC0\x80"); + + } else { + $this->_data .= $str; + } + + $this->_position = strlen($this->_data); + } + + + /** + * Reads binary data from the current position in the file + * and advances the file pointer. + * + * @return string + */ + public function readBinary() + { + $length = $this->readVInt(); + $returnValue = substr($this->_data, $this->_position, $length); + $this->_position += $length; + return $returnValue; + } +} + diff --git a/search/Zend/Search/TODO.txt b/search/Zend/Search/TODO.txt index 06f7b48792e..799a19e9607 100644 --- a/search/Zend/Search/TODO.txt +++ b/search/Zend/Search/TODO.txt @@ -1,14 +1,7 @@ @todo -- Improve API: fix ZSearchMultiTermQuery($terms, $signs); - -- Analysis and indexing engine - -- Additional queries: phrase, wildcard, proximity, and range +- Additional queries: wildcard, proximity, and range - Better class-level docblocks (most functions okay) -- Some Windows issues(?) during indexing - -- Finish renaming classes to PEAR-like conventions