try update Zend directory to 1.9 equivalent

search/Zend/Search/Exception.php

@@ -14,7 +14,7 @@
  *
  * @category   Zend
  * @package    Zend_Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
@@ -22,13 +22,13 @@
 /**
  * Framework base exception
  */
-require_once 'Zend/Exception.php';
+require_once $CFG->dirroot.'/search/Zend/Exception.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Exception extends Zend_Exception

search/Zend/Search/Lucene.php

@@ -14,53 +14,78 @@
  *
  * @category   Zend
  * @package    Zend_Search_Lucene
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Exception */
-require_once 'Zend/Search/Lucene/Exception.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
 
 /** Zend_Search_Lucene_Document */
-require_once 'Zend/Search/Lucene/Document.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Document.php';
+
+/** Zend_Search_Lucene_Document_Html */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Document/Html.php';
 
 /** Zend_Search_Lucene_Storage_Directory */
-require_once 'Zend/Search/Lucene/Storage/Directory/Filesystem.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/Directory/Filesystem.php';
+
+/** Zend_Search_Lucene_Storage_File_Memory */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/File/Memory.php';
 
 /** Zend_Search_Lucene_Index_Term */
-require_once 'Zend/Search/Lucene/Index/Term.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php';
 
 /** Zend_Search_Lucene_Index_TermInfo */
-require_once 'Zend/Search/Lucene/Index/TermInfo.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/TermInfo.php';
 
 /** Zend_Search_Lucene_Index_SegmentInfo */
-require_once 'Zend/Search/Lucene/Index/SegmentInfo.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php';
 
 /** Zend_Search_Lucene_Index_FieldInfo */
-require_once 'Zend/Search/Lucene/Index/FieldInfo.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/FieldInfo.php';
 
 /** Zend_Search_Lucene_Index_Writer */
-require_once 'Zend/Search/Lucene/Index/Writer.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Writer.php';
 
 /** Zend_Search_Lucene_Search_QueryParser */
-require_once 'Zend/Search/Lucene/Search/QueryParser.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParser.php';
 
 /** Zend_Search_Lucene_Search_QueryHit */
-require_once 'Zend/Search/Lucene/Search/QueryHit.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryHit.php';
 
 /** Zend_Search_Lucene_Search_Similarity */
-require_once 'Zend/Search/Lucene/Search/Similarity.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Similarity.php';
+
+/** Zend_Search_Lucene_Index_SegmentInfoPriorityQueue */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php';
+
+
+/** Zend_Search_Lucene_Interface */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Interface.php';
+
+/** Zend_Search_Lucene_Proxy */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Proxy.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
-class Zend_Search_Lucene
+class Zend_Search_Lucene implements Zend_Search_Lucene_Interface
 {
+    /**
+     * Default field name for search
+     *
+     * Null means search through all fields
+     *
+     * @var string
+     */
+    private static $_defaultSearchField = null;
+
     /**
      * File system adapter.
      *
@@ -103,6 +128,51 @@ class Zend_Search_Lucene
      */
     private $_hasChanges = false;
 
+
+    /**
+     * Index lock object
+     *
+     * @var Zend_Search_Lucene_Storage_File
+     */
+    private $_lock;
+
+    /**
+     * Signal, that index is already closed, changes are fixed and resources are cleaned up
+     *
+     * @var boolean
+     */
+    private $_closed = false;
+
+    /**
+     * Number of references to the index object
+     *
+     * @var integer
+     */
+    private $_refCount = 0;
+
+
+    /**
+     * Create index
+     *
+     * @param mixed $directory
+     * @return Zend_Search_Lucene_Interface
+     */
+    public static function create($directory)
+    {
+        return new Zend_Search_Lucene_Proxy(new Zend_Search_Lucene($directory, true));
+    }
+
+    /**
+     * Open index
+     *
+     * @param mixed $directory
+     * @return Zend_Search_Lucene_Interface
+     */
+    public static function open($directory)
+    {
+        return new Zend_Search_Lucene_Proxy(new Zend_Search_Lucene($directory, false));
+    }
+
     /**
      * Opens the index.
      *
@@ -126,13 +196,32 @@ class Zend_Search_Lucene
             $this->_closeDirOnExit = true;
         }
 
+
+        // Get a shared lock to the index
+        $this->_lock = $this->_directory->createFile('index.lock');
+
+        $this->_segmentInfos = array();
+
         if ($create) {
-            $this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory, true);
+            // Throw an exception if index is under processing now
+            if (!$this->_lock->lock(LOCK_EX, true)) {
+                throw new Zend_Search_Lucene_Exception('Can\'t create index. It\'s under processing now');
+            }
+
+            // Writer will create segments file for empty segments list
+            $this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory, $this->_segmentInfos, true);
+
+            if (!$this->_lock->lock(LOCK_SH)) {
+                throw new Zend_Search_Lucene_Exception('Can\'t reduce lock level from Exclusive to Shared');
+            }
         } else {
+            // Wait if index is under switching from one set of segments to another (Index_Writer::_updateSegments())
+            if (!$this->_lock->lock(LOCK_SH)) {
+                throw new Zend_Search_Lucene_Exception('Can\'t obtain shared index lock');
+            }
             $this->_writer = null;
         }
 
-        $this->_segmentInfos = array();
 
         $segmentsFile = $this->_directory->getFileObject('segments');
 
@@ -143,9 +232,10 @@ class Zend_Search_Lucene
         }
 
         // read version
-        $segmentsFile->readLong();
+        // $segmentsFile->readLong();
+        $segmentsFile->readInt(); $segmentsFile->readInt();
 
-        // read counter
+        // read segment name counter
         $segmentsFile->readInt();
 
         $segments = $segmentsFile->readInt();
@@ -158,35 +248,83 @@ class Zend_Search_Lucene
             $segSize = $segmentsFile->readInt();
             $this->_docCount += $segSize;
 
-            $this->_segmentInfos[$count] =
+            $this->_segmentInfos[] =
                                 new Zend_Search_Lucene_Index_SegmentInfo($segName,
                                                                          $segSize,
                                                                          $this->_directory);
         }
     }
 
+    /**
+     * Close current index and free resources
+     */
+    private function _close()
+    {
+        if ($this->_closed) {
+            // index is already closed and resources are cleaned up
+            return;
+        }
+
+        $this->commit();
+
+        // Free shared lock
+        $this->_lock->unlock();
+
+        if ($this->_closeDirOnExit) {
+            $this->_directory->close();
+        }
+
+        $this->_directory    = null;
+        $this->_writer       = null;
+        $this->_segmentInfos = null;
+
+        $this->_closed = true;
+    }
+
+    /**
+     * Add reference to the index object
+     *
+     * @internal
+     */
+    public function addReference()
+    {
+        $this->_refCount++;
+    }
+
+    /**
+     * Remove reference from the index object
+     *
+     * When reference count becomes zero, index is closed and resources are cleaned up
+     *
+     * @internal
+     */
+    public function removeReference()
+    {
+        $this->_refCount--;
+
+        if ($this->_refCount == 0) {
+            $this->_close();
+        }
+    }
 
     /**
      * Object destructor
      */
     public function __destruct()
     {
-        $this->commit();
-
-        if ($this->_closeDirOnExit) {
-            $this->_directory->close();
-        }
+        $this->_close();
     }
 
     /**
      * Returns an instance of Zend_Search_Lucene_Index_Writer for the index
      *
+     * @internal
      * @return Zend_Search_Lucene_Index_Writer
      */
     public function getIndexWriter()
     {
         if (!$this->_writer instanceof Zend_Search_Lucene_Index_Writer) {
-            $this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory);
+            $this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory, $this->_segmentInfos);
         }
 
         return $this->_writer;
@@ -205,7 +343,7 @@ class Zend_Search_Lucene
 
 
     /**
-     * Returns the total number of documents in this index.
+     * Returns the total number of documents in this index (including deleted documents).
      *
      * @return integer
      */
@@ -214,6 +352,192 @@ class Zend_Search_Lucene
         return $this->_docCount;
     }
 
+    /**
+     * Returns one greater than the largest possible document number.
+     * This may be used to, e.g., determine how big to allocate a structure which will have
+     * an element for every document number in an index.
+     *
+     * @return integer
+     */
+    public function maxDoc()
+    {
+        return $this->count();
+    }
+
+    /**
+     * Returns the total number of non-deleted documents in this index.
+     *
+     * @return integer
+     */
+    public function numDocs()
+    {
+        $numDocs = 0;
+
+        foreach ($this->_segmentInfos as $segmentInfo) {
+            $numDocs += $segmentInfo->numDocs();
+        }
+
+        return $numDocs;
+    }
+
+    /**
+     * Checks, that document is deleted
+     *
+     * @param integer $id
+     * @return boolean
+     * @throws Zend_Search_Lucene_Exception    Exception is thrown if $id is out of the range
+     */
+    public function isDeleted($id)
+    {
+        if ($id >= $this->_docCount) {
+            throw new Zend_Search_Lucene_Exception('Document id is out of the range.');
+        }
+
+        $segmentStartId = 0;
+        foreach ($this->_segmentInfos as $segmentInfo) {
+            if ($segmentStartId + $segmentInfo->count() > $id) {
+                break;
+            }
+
+            $segmentStartId += $segmentInfo->count();
+        }
+
+        return $segmentInfo->isDeleted($id - $segmentStartId);
+    }
+
+    /**
+     * Set default search field.
+     *
+     * Null means, that search is performed through all fields by default
+     *
+     * Default value is null
+     *
+     * @param string $fieldName
+     */
+    public static function setDefaultSearchField($fieldName)
+    {
+        self::$_defaultSearchField = $fieldName;
+    }
+
+    /**
+     * Get default search field.
+     *
+     * Null means, that search is performed through all fields by default
+     *
+     * @return string
+     */
+    public static function getDefaultSearchField()
+    {
+        return self::$_defaultSearchField;
+    }
+
+    /**
+     * Retrieve index maxBufferedDocs option
+     *
+     * maxBufferedDocs is a minimal number of documents required before
+     * the buffered in-memory documents are written into a new Segment
+     *
+     * Default value is 10
+     *
+     * @return integer
+     */
+    public function getMaxBufferedDocs()
+    {
+        return $this->getIndexWriter()->maxBufferedDocs;
+    }
+
+    /**
+     * Set index maxBufferedDocs option
+     *
+     * maxBufferedDocs is a minimal number of documents required before
+     * the buffered in-memory documents are written into a new Segment
+     *
+     * Default value is 10
+     *
+     * @param integer $maxBufferedDocs
+     */
+    public function setMaxBufferedDocs($maxBufferedDocs)
+    {
+        $this->getIndexWriter()->maxBufferedDocs = $maxBufferedDocs;
+    }
+
+    /**
+     * Retrieve index maxMergeDocs option
+     *
+     * maxMergeDocs is a largest number of documents ever merged by addDocument().
+     * Small values (e.g., less than 10,000) are best for interactive indexing,
+     * as this limits the length of pauses while indexing to a few seconds.
+     * Larger values are best for batched indexing and speedier searches.
+     *
+     * Default value is PHP_INT_MAX
+     *
+     * @return integer
+     */
+    public function getMaxMergeDocs()
+    {
+        return $this->getIndexWriter()->maxMergeDocs;
+    }
+
+    /**
+     * Set index maxMergeDocs option
+     *
+     * maxMergeDocs is a largest number of documents ever merged by addDocument().
+     * Small values (e.g., less than 10,000) are best for interactive indexing,
+     * as this limits the length of pauses while indexing to a few seconds.
+     * Larger values are best for batched indexing and speedier searches.
+     *
+     * Default value is PHP_INT_MAX
+     *
+     * @param integer $maxMergeDocs
+     */
+    public function setMaxMergeDocs($maxMergeDocs)
+    {
+        $this->getIndexWriter()->maxMergeDocs = $maxMergeDocs;
+    }
+
+    /**
+     * Retrieve index mergeFactor option
+     *
+     * mergeFactor determines how often segment indices are merged by addDocument().
+     * With smaller values, less RAM is used while indexing,
+     * and searches on unoptimized indices are faster,
+     * but indexing speed is slower.
+     * With larger values, more RAM is used during indexing,
+     * and while searches on unoptimized indices are slower,
+     * indexing is faster.
+     * Thus larger values (> 10) are best for batch index creation,
+     * and smaller values (< 10) for indices that are interactively maintained.
+     *
+     * Default value is 10
+     *
+     * @return integer
+     */
+    public function getMergeFactor()
+    {
+        return $this->getIndexWriter()->mergeFactor;
+    }
+
+    /**
+     * Set index mergeFactor option
+     *
+     * mergeFactor determines how often segment indices are merged by addDocument().
+     * With smaller values, less RAM is used while indexing,
+     * and searches on unoptimized indices are faster,
+     * but indexing speed is slower.
+     * With larger values, more RAM is used during indexing,
+     * and while searches on unoptimized indices are slower,
+     * indexing is faster.
+     * Thus larger values (> 10) are best for batch index creation,
+     * and smaller values (< 10) for indices that are interactively maintained.
+     *
+     * Default value is 10
+     *
+     * @param integer $maxMergeDocs
+     */
+    public function setMergeFactor($mergeFactor)
+    {
+        $this->getIndexWriter()->mergeFactor = $mergeFactor;
+    }
 
     /**
      * Performs a query against the index and returns an array
@@ -221,7 +545,8 @@ class Zend_Search_Lucene
      * Input is a string or Zend_Search_Lucene_Search_Query.
      *
      * @param mixed $query
-     * @return array ZSearchHit
+     * @return array Zend_Search_Lucene_Search_QueryHit
+     * @throws Zend_Search_Lucene_Exception
      */
     public function find($query)
     {
@@ -235,22 +560,115 @@ class Zend_Search_Lucene
 
         $this->commit();
 
-        $hits = array();
+        $hits   = array();
         $scores = array();
+        $ids    = array();
 
-        $docNum = $this->count();
-        for( $count=0; $count < $docNum; $count++ ) {
-            $docScore = $query->score( $count, $this);
+        $query = $query->rewrite($this)->optimize($this);
+
+        $query->execute($this);
+
+        $topScore = 0;
+
+        foreach ($query->matchedDocs() as $id => $num) {
+            $docScore = $query->score($id, $this);
             if( $docScore != 0 ) {
                 $hit = new Zend_Search_Lucene_Search_QueryHit($this);
-                $hit->id = $count;
+                $hit->id = $id;
                 $hit->score = $docScore;
 
-                $hits[] = $hit;
+                $hits[]   = $hit;
+                $ids[]    = $id;
                 $scores[] = $docScore;
+
+                if ($docScore > $topScore) {
+                    $topScore = $docScore;
+                }
             }
         }
-        array_multisort($scores, SORT_DESC, SORT_REGULAR, $hits);
+
+        if (count($hits) == 0) {
+            // skip sorting, which may cause a error on empty index
+        	return array();
+        }
+
+        if ($topScore > 1) {
+            foreach ($hits as $hit) {
+                $hit->score /= $topScore;
+            }
+        }
+
+        if (func_num_args() == 1) {
+            // sort by scores
+            array_multisort($scores, SORT_DESC, SORT_NUMERIC,
+                            $ids,    SORT_ASC,  SORT_NUMERIC,
+                            $hits);
+        } else {
+            // sort by given field names
+
+            $argList    = func_get_args();
+            $fieldNames = $this->getFieldNames();
+            $sortArgs   = array();
+
+            for ($count = 1; $count < count($argList); $count++) {
+                $fieldName = $argList[$count];
+
+                if (!is_string($fieldName)) {
+                    throw new Zend_Search_Lucene_Exception('Field name must be a string.');
+                }
+
+                if (!in_array($fieldName, $fieldNames)) {
+                    throw new Zend_Search_Lucene_Exception('Wrong field name.');
+                }
+
+                $valuesArray = array();
+                foreach ($hits as $hit) {
+                    try {
+                        $value = $hit->getDocument()->getFieldValue($fieldName);
+                    } catch (Zend_Search_Lucene_Exception $e) {
+                        if (strpos($e->getMessage(), 'not found') === false) {
+                            throw $e;
+                        } else {
+                            $value = null;
+                        }
+                    }
+
+                    $valuesArray[] = $value;
+                }
+
+                $sortArgs[] = $valuesArray;
+
+                if ($count + 1 < count($argList)  &&  is_integer($argList[$count+1])) {
+                    $count++;
+                    $sortArgs[] = $argList[$count];
+
+                    if ($count + 1 < count($argList)  &&  is_integer($argList[$count+1])) {
+                        $count++;
+                        $sortArgs[] = $argList[$count];
+                    } else {
+                        if ($argList[$count] == SORT_ASC  || $argList[$count] == SORT_DESC) {
+                            $sortArgs[] = SORT_REGULAR;
+                        } else {
+                            $sortArgs[] = SORT_ASC;
+                        }
+                    }
+                } else {
+                    $sortArgs[] = SORT_ASC;
+                    $sortArgs[] = SORT_REGULAR;
+                }
+            }
+
+            // Sort by id's if values are equal
+            $sortArgs[] = $ids;
+            $sortArgs[] = SORT_ASC;
+            $sortArgs[] = SORT_NUMERIC;
+
+            // Array to be sorted
+            $sortArgs[] = &$hits;
+
+            // Do sort
+            call_user_func_array('array_multisort', $sortArgs);
+        }
 
         return $hits;
     }
@@ -290,41 +708,45 @@ class Zend_Search_Lucene
             throw new Zend_Search_Lucene_Exception('Document id is out of the range.');
         }
 
-        $segCount = 0;
-        $nextSegmentStartId = $this->_segmentInfos[ 0 ]->count();
-        while( $nextSegmentStartId <= $id ) {
-               $segCount++;
-               $nextSegmentStartId += $this->_segmentInfos[ $segCount ]->count();
-        }
-        $segmentStartId = $nextSegmentStartId - $this->_segmentInfos[ $segCount ]->count();
+        $segmentStartId = 0;
+        foreach ($this->_segmentInfos as $segmentInfo) {
+            if ($segmentStartId + $segmentInfo->count() > $id) {
+                break;
+            }
 
-        $fdxFile = $this->_segmentInfos[ $segCount ]->openCompoundFile('.fdx');
+            $segmentStartId += $segmentInfo->count();
+        }
+
+        $fdxFile = $segmentInfo->openCompoundFile('.fdx');
         $fdxFile->seek( ($id-$segmentStartId)*8, SEEK_CUR );
         $fieldValuesPosition = $fdxFile->readLong();
 
-        $fdtFile = $this->_segmentInfos[ $segCount ]->openCompoundFile('.fdt');
-        $fdtFile->seek( $fieldValuesPosition, SEEK_CUR );
+        $fdtFile = $segmentInfo->openCompoundFile('.fdt');
+        $fdtFile->seek($fieldValuesPosition, SEEK_CUR);
         $fieldCount = $fdtFile->readVInt();
 
         $doc = new Zend_Search_Lucene_Document();
-        for( $count = 0; $count < $fieldCount; $count++ ) {
+        for ($count = 0; $count < $fieldCount; $count++) {
             $fieldNum = $fdtFile->readVInt();
             $bits = $fdtFile->readByte();
 
-            $fieldInfo = $this->_segmentInfos[ $segCount ]->getField($fieldNum);
+            $fieldInfo = $segmentInfo->getField($fieldNum);
 
-            if( !($bits & 2) ) { // Text data
+            if (!($bits & 2)) { // Text data
                 $field = new Zend_Search_Lucene_Field($fieldInfo->name,
                                                       $fdtFile->readString(),
+                                                      'UTF-8',
                                                       true,
                                                       $fieldInfo->isIndexed,
                                                       $bits & 1 );
-            } else {
+            } else {            // Binary data
                 $field = new Zend_Search_Lucene_Field($fieldInfo->name,
                                                       $fdtFile->readBinary(),
+                                                      '',
                                                       true,
                                                       $fieldInfo->isIndexed,
-                                                      $bits & 1 );
+                                                      $bits & 1,
+                                                      true );
             }
 
             $doc->addField($field);
@@ -335,7 +757,26 @@ class Zend_Search_Lucene
 
 
     /**
-     * Returns an array of all the documents which contain term.
+     * Returns true if index contain documents with specified term.
+     *
+     * Is used for query optimization.
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @return boolean
+     */
+    public function hasTerm(Zend_Search_Lucene_Index_Term $term)
+    {
+        foreach ($this->_segmentInfos as $segInfo) {
+            if ($segInfo->getTermInfo($term) instanceof Zend_Search_Lucene_Index_TermInfo) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Returns IDs of all the documents containing term.
      *
      * @param Zend_Search_Lucene_Index_Term $term
      * @return array
@@ -376,9 +817,29 @@ class Zend_Search_Lucene
     }
 
 
+    /**
+     * Returns an array of all term freqs.
+     * Result array structure: array(docId => freq, ...)
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @return integer
+     */
+    public function termFreqs(Zend_Search_Lucene_Index_Term $term)
+    {
+        $result = array();
+        $segmentStartDocId = 0;
+        foreach ($this->_segmentInfos as $segmentInfo) {
+            $result += $segmentInfo->termFreqs($term, $segmentStartDocId);
+
+            $segmentStartDocId += $segmentInfo->count();
+        }
+
+        return $result;
+    }
+
     /**
      * Returns an array of all term positions in the documents.
-     * Return array structure: array( docId => array( pos1, pos2, ...), ...)
+     * Result array structure: array(docId => array(pos1, pos2, ...), ...)
      *
      * @param Zend_Search_Lucene_Index_Term $term
      * @return array
@@ -387,45 +848,10 @@ class Zend_Search_Lucene
     {
         $result = array();
         $segmentStartDocId = 0;
-        foreach( $this->_segmentInfos as $segInfo ) {
-            $termInfo = $segInfo->getTermInfo($term);
+        foreach ($this->_segmentInfos as $segmentInfo) {
+            $result += $segmentInfo->termPositions($term, $segmentStartDocId);
 
-            if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) {
-                $segmentStartDocId += $segInfo->count();
-                continue;
-            }
-
-            $frqFile = $segInfo->openCompoundFile('.frq');
-            $frqFile->seek($termInfo->freqPointer,SEEK_CUR);
-            $freqs = array();
-            $docId = 0;
-
-            for( $count = 0; $count < $termInfo->docFreq; $count++ ) {
-                $docDelta = $frqFile->readVInt();
-                if( $docDelta % 2 == 1 ) {
-                    $docId += ($docDelta-1)/2;
-                    $freqs[ $docId ] = 1;
-                } else {
-                    $docId += $docDelta/2;
-                    $freqs[ $docId ] = $frqFile->readVInt();
-                }
-            }
-
-            $prxFile = $segInfo->openCompoundFile('.prx');
-            $prxFile->seek($termInfo->proxPointer,SEEK_CUR);
-            foreach ($freqs as $docId => $freq) {
-                $termPosition = 0;
-                $positions = array();
-
-                for ($count = 0; $count < $freq; $count++ ) {
-                    $termPosition += $prxFile->readVInt();
-                    $positions[] = $termPosition;
-                }
-
-                $result[ $segmentStartDocId + $docId ] = $positions;
-            }
-
-            $segmentStartDocId += $segInfo->count();
+            $segmentStartDocId += $segmentInfo->count();
         }
 
         return $result;
@@ -468,9 +894,9 @@ class Zend_Search_Lucene
      *
      * @param integer $id
      * @param string $fieldName
-     * @return Zend_Search_Lucene_Document
+     * @return float
      */
-    public function norm( $id, $fieldName )
+    public function norm($id, $fieldName)
     {
         if ($id >= $this->_docCount) {
             return null;
@@ -527,16 +953,17 @@ class Zend_Search_Lucene
             throw new Zend_Search_Lucene_Exception('Document id is out of the range.');
         }
 
-        $segCount = 0;
-        $nextSegmentStartId = $this->_segmentInfos[ 0 ]->count();
-        while( $nextSegmentStartId <= $id ) {
-               $segCount++;
-               $nextSegmentStartId += $this->_segmentInfos[ $segCount ]->count();
+        $segmentStartId = 0;
+        foreach ($this->_segmentInfos as $segmentInfo) {
+            if ($segmentStartId + $segmentInfo->count() > $id) {
+                break;
+            }
+
+            $segmentStartId += $segmentInfo->count();
         }
+        $segmentInfo->delete($id - $segmentStartId);
 
         $this->_hasChanges = true;
-        $segmentStartId = $nextSegmentStartId - $this->_segmentInfos[ $segCount ]->count();
-        $this->_segmentInfos[ $segCount ]->delete($id - $segmentStartId);
     }
 
 
@@ -548,18 +975,26 @@ class Zend_Search_Lucene
      */
     public function addDocument(Zend_Search_Lucene_Document $document)
     {
-        if (!$this->_writer instanceof Zend_Search_Lucene_Index_Writer) {
-            $this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory);
-        }
-
-        $this->_writer->addDocument($document);
+        $this->getIndexWriter()->addDocument($document);
+        $this->_docCount++;
     }
 
 
+    /**
+     * Update document counter
+     */
+    private function _updateDocCount()
+    {
+        $this->_docCount = 0;
+        foreach ($this->_segmentInfos as $segInfo) {
+            $this->_docCount += $segInfo->count();
+        }
+    }
+
     /**
      * Commit changes resulting from delete() or undeleteAll() operations.
      *
-     * @todo delete() and undeleteAll processing.
+     * @todo undeleteAll processing.
      */
     public function commit()
     {
@@ -572,38 +1007,73 @@ class Zend_Search_Lucene
         }
 
         if ($this->_writer !== null) {
-            foreach ($this->_writer->commit() as $segmentName => $segmentInfo) {
-                if ($segmentInfo !== null) {
-                    $this->_segmentInfos[] = $segmentInfo;
-                    $this->_docCount += $segmentInfo->count();
-                } else {
-                    foreach ($this->_segmentInfos as $segId => $segInfo) {
-                        if ($segInfo->getName() == $segmentName) {
-                            unset($this->_segmentInfos[$segId]);
-                        }
-                    }
-                }
+            $this->_writer->commit();
+
+            $this->_updateDocCount();
+        }
+    }
+
+
+    /**
+     * Optimize index.
+     *
+     * Merges all segments into one
+     */
+    public function optimize()
+    {
+        // Commit changes if any changes have been made
+        $this->commit();
+
+        if (count($this->_segmentInfos) > 1 || $this->hasDeletions()) {
+            $this->getIndexWriter()->optimize();
+            $this->_updateDocCount();
+        }
+    }
+
+
+    /**
+     * Returns an array of all terms in this index.
+     *
+     * @return array
+     */
+    public function terms()
+    {
+        $result = array();
+
+        $segmentInfoQueue = new Zend_Search_Lucene_Index_SegmentInfoPriorityQueue();
+
+        foreach ($this->_segmentInfos as $segmentInfo) {
+            $segmentInfo->reset();
+
+            // Skip "empty" segments
+            if ($segmentInfo->currentTerm() !== null) {
+                $segmentInfoQueue->put($segmentInfo);
             }
         }
+
+        while (($segmentInfo = $segmentInfoQueue->pop()) !== null) {
+            if ($segmentInfoQueue->top() === null ||
+                $segmentInfoQueue->top()->currentTerm()->key() !=
+                            $segmentInfo->currentTerm()->key()) {
+                // We got new term
+                $result[] = $segmentInfo->currentTerm();
+            }
+
+            $segmentInfo->nextTerm();
+            // check, if segment dictionary is finished
+            if ($segmentInfo->currentTerm() !== null) {
+                // Put segment back into the priority queue
+                $segmentInfoQueue->put($segmentInfo);
+            }
+        }
+
+        return $result;
     }
 
 
     /*************************************************************************
     @todo UNIMPLEMENTED
     *************************************************************************/
-
-    /**
-     * Returns an array of all terms in this index.
-     *
-     * @todo Implementation
-     * @return array
-     */
-    public function terms()
-    {
-        return array();
-    }
-
-
     /**
      * Undeletes all documents currently marked as deleted in this index.
      *
@@ -611,4 +1081,4 @@ class Zend_Search_Lucene
      */
     public function undeleteAll()
     {}
-}
+}

search/Zend/Search/Lucene/Analysis/Analyzer.php

@@ -15,20 +15,37 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Analysis
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Analysis_Token */
-require_once 'Zend/Search/Lucene/Analysis/Token.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Token.php';
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8 */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php';
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php';
 
 /** Zend_Search_Lucene_Analysis_Analyzer_Common_Text */
-require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php';
 
 /** Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive */
-require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php';
 
+/** Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php';
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php';
+
+/** Zend_Search_Lucene_Analysis_TokenFilter_StopWords */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php';
+
+/** Zend_Search_Lucene_Analysis_TokenFilter_ShortWords */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php';
 
 
 /**
@@ -44,7 +61,7 @@ require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.p
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Analysis
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
@@ -55,16 +72,73 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer
      *
      * @var Zend_Search_Lucene_Analysis_Analyzer
      */
-    static private $_defaultImpl;
+    private static $_defaultImpl;
+
+    /**
+     * Input string
+     *
+     * @var string
+     */
+    protected $_input = null;
+
+    /**
+     * Input string encoding
+     *
+     * @var string
+     */
+    protected $_encoding = '';
 
     /**
      * Tokenize text to a terms
      * Returns array of Zend_Search_Lucene_Analysis_Token objects
      *
+     * Tokens are returned in UTF-8 (internal Zend_Search_Lucene encoding)
+     *
      * @param string $data
      * @return array
      */
-    abstract public function tokenize($data);
+    public function tokenize($data, $encoding = 'UTF-8')
+    {
+        $this->setInput($data, $encoding);
+        $tokenList = array();
+        while (($nextToken = $this->nextToken()) !== null) {
+            $tokenList[] = $nextToken;
+        }
+
+        return $tokenList;
+    }
+
+
+    /**
+     * Tokenization stream API
+     * Set input
+     *
+     * @param string $data
+     */
+    public function setInput($data, $encoding = '')
+    {
+        $this->_input    = $data;
+        $this->_encoding = $encoding;
+        $this->reset();
+    }
+
+    /**
+     * Reset token stream
+     */
+    abstract public function reset();
+
+    /**
+     * Tokenization stream API
+     * Get next token
+     * Returns null at the end of stream
+     *
+     * Tokens are returned in UTF-8 (internal Zend_Search_Lucene encoding)
+     *
+     * @return Zend_Search_Lucene_Analysis_Token|null
+     */
+    abstract public function nextToken();
+
+
 
 
     /**
@@ -72,7 +146,7 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer
      *
      * @param Zend_Search_Lucene_Analysis_Analyzer $similarity
      */
-    static public function setDefault(Zend_Search_Lucene_Analysis_Analyzer $analyzer)
+    public static function setDefault(Zend_Search_Lucene_Analysis_Analyzer $analyzer)
     {
         self::$_defaultImpl = $analyzer;
     }
@@ -83,14 +157,13 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer
      *
      * @return Zend_Search_Lucene_Analysis_Analyzer
      */
-    static public function getDefault()
+    public static function getDefault()
     {
         if (!self::$_defaultImpl instanceof Zend_Search_Lucene_Analysis_Analyzer) {
-            self::$_defaultImpl = new Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive();
+            self::$_defaultImpl = new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8();
         }
 
         return self::$_defaultImpl;
     }
-
 }
 

search/Zend/Search/Lucene/Analysis/Analyzer/Common.php

@@ -15,13 +15,13 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Analysis
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Analysis_Analyzer */
-require_once 'Zend/Search/Lucene/Analysis/Analyzer.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php';
 
 
 /**
@@ -34,7 +34,7 @@ require_once 'Zend/Search/Lucene/Analysis/Analyzer.php';
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Analysis
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 abstract class Zend_Search_Lucene_Analysis_Analyzer_Common extends Zend_Search_Lucene_Analysis_Analyzer
@@ -58,7 +58,7 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer_Common extends Zend_Search_L
     }
 
     /**
-     * Apply filters to the token.
+     * Apply filters to the token. Can return null when the token was removed.
      *
      * @param Zend_Search_Lucene_Analysis_Token $token
      * @return Zend_Search_Lucene_Analysis_Token
@@ -67,6 +67,11 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer_Common extends Zend_Search_L
     {
         foreach ($this->_filters as $filter) {
             $token = $filter->normalize($token);
+
+            // resulting token can be null if the filter removed it
+            if (is_null($token)) {
+                return null;
+            }
         }
 
         return $token;

search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php

@@ -15,64 +15,79 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Analysis
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Analysis_Analyzer_Common */
-require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Analysis
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 class Zend_Search_Lucene_Analysis_Analyzer_Common_Text extends Zend_Search_Lucene_Analysis_Analyzer_Common
 {
     /**
-     * Tokenize text to a terms
-     * Returns array of Zend_Search_Lucene_Analysis_Token objects
+     * Current position in a stream
      *
-     * @param string $data
-     * @return array
+     * @var integer
      */
-    public function tokenize($data)
+    private $_position;
+
+    /**
+     * Reset token stream
+     */
+    public function reset()
     {
-        $tokenStream = array();
+        $this->_position = 0;
 
-        $position = 0;
-        while ($position < strlen($data)) {
-            // skip white space
-            while ($position < strlen($data) && !ctype_alpha( $data{$position} )) {
-                $position++;
-            }
-
-            $termStartPosition = $position;
-
-            // read token
-            while ($position < strlen($data) && ctype_alpha( $data{$position} )) {
-                $position++;
-            }
-
-            // Empty token, end of stream.
-            if ($position == $termStartPosition) {
-                break;
-            }
-
-            $token = new Zend_Search_Lucene_Analysis_Token(substr($data,
-                                             $termStartPosition,
-                                             $position-$termStartPosition),
-                                      $termStartPosition,
-                                      $position);
-            $tokenStream[] = $this->normalize($token);
+        if ($this->_input === null) {
+            return;
         }
 
-        return $tokenStream;
+        // convert input into ascii
+        $this->_input = iconv($this->_encoding, 'ASCII//TRANSLIT', $this->_input);
+        $this->_encoding = 'ASCII';
+    }
+
+    /**
+     * Tokenization stream API
+     * Get next token
+     * Returns null at the end of stream
+     *
+     * @return Zend_Search_Lucene_Analysis_Token|null
+     */
+    public function nextToken()
+    {
+        if ($this->_input === null) {
+            return null;
+        }
+
+
+        do {
+            if (! preg_match('/[a-zA-Z]+/', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_position)) {
+                // It covers both cases a) there are no matches (preg_match(...) === 0)
+                // b) error occured (preg_match(...) === FALSE)
+            	return null;
+            }
+
+            $str = $match[0][0];
+            $pos = $match[0][1];
+            $endpos = $pos + strlen($str);
+
+            $this->_position = $endpos;
+
+            $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($str, $pos, $endpos));
+        } while ($token === null); // try again if token is skipped
+
+        return $token;
     }
 }
 

search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php

@@ -15,23 +15,23 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Analysis
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Analysis_Analyzer_Common_Text */
-require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php';
 
 /** Zend_Search_Lucene_Analysis_TokenFilter_LowerCase */
-require_once 'Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Analysis
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 

search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php

@@ -0,0 +1,92 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+class Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum extends Zend_Search_Lucene_Analysis_Analyzer_Common
+{
+    /**
+     * Current position in a stream
+     *
+     * @var integer
+     */
+    private $_position;
+
+    /**
+     * Reset token stream
+     */
+    public function reset()
+    {
+        $this->_position = 0;
+
+        if ($this->_input === null) {
+            return;
+        }
+
+        // convert input into ascii
+        $this->_input = iconv($this->_encoding, 'ASCII//TRANSLIT', $this->_input);
+        $this->_encoding = 'ASCII';
+    }
+
+    /**
+     * Tokenization stream API
+     * Get next token
+     * Returns null at the end of stream
+     *
+     * @return Zend_Search_Lucene_Analysis_Token|null
+     */
+    public function nextToken()
+    {
+        if ($this->_input === null) {
+            return null;
+        }
+
+        do {
+            if (! preg_match('/[a-zA-Z0-9]+/', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_position)) {
+                // It covers both cases a) there are no matches (preg_match(...) === 0)
+                // b) error occured (preg_match(...) === FALSE)
+            	return null;
+            }
+
+            $str = $match[0][0];
+            $pos = $match[0][1];
+            $endpos = $pos + strlen($str);
+
+            $this->_position = $endpos;
+
+            $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($str, $pos, $endpos));
+        } while ($token === null); // try again if token is skipped
+
+        return $token;
+    }
+}
+

search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php

@@ -0,0 +1,46 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php';
+
+/** Zend_Search_Lucene_Analysis_TokenFilter_LowerCase */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+class Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive extends Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum
+{
+    public function __construct()
+    {
+        $this->addFilter(new Zend_Search_Lucene_Analysis_TokenFilter_LowerCase());
+    }
+}
+

search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php

@@ -0,0 +1,169 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+class Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8 extends Zend_Search_Lucene_Analysis_Analyzer_Common
+{
+    /**
+     * Current char position in an UTF-8 stream
+     *
+     * @var integer
+     */
+    private $_position;
+
+    /**
+     * Current binary position in an UTF-8 stream
+     *
+     * @var integer
+     */
+    private $_bytePosition;
+
+    /**
+     * Stream length
+     *
+     * @var integer
+     */
+    private $_streamLength;
+
+    /**
+     * Reset token stream
+     */
+    public function reset()
+    {
+        $this->_position     = 0;
+        $this->_bytePosition = 0;
+
+        // convert input into UTF-8
+        if (strcasecmp($this->_encoding, 'utf8' ) != 0  &&
+            strcasecmp($this->_encoding, 'utf-8') != 0 ) {
+                $this->_input = iconv($this->_encoding, 'UTF-8', $this->_input);
+                $this->_encoding = 'UTF-8';
+        }
+
+        // Get UTF-8 string length.
+        // It also checks if it's a correct utf-8 string
+        $this->_streamLength = iconv_strlen($this->_input, 'UTF-8');
+    }
+
+    /**
+     * Check, that character is a letter
+     *
+     * @param string $char
+     * @return boolean
+     */
+    private static function _isAlpha($char)
+    {
+        if (strlen($char) > 1) {
+            // It's an UTF-8 character
+            return true;
+        }
+
+        return ctype_alpha($char);
+    }
+
+    /**
+     * Get next UTF-8 char
+     *
+     * @param string $char
+     * @return boolean
+     */
+    private function _nextChar()
+    {
+        $char = $this->_input[$this->_bytePosition++];
+
+        if (( ord($char) & 0xC0 ) == 0xC0) {
+            $addBytes = 1;
+            if (ord($char) & 0x20 ) {
+                $addBytes++;
+                if (ord($char) & 0x10 ) {
+                    $addBytes++;
+                }
+            }
+            $char .= substr($this->_input, $this->_bytePosition, $addBytes);
+            $this->_bytePosition += $addBytes;
+        }
+
+        $this->_position++;
+
+        return $char;
+    }
+
+    /**
+     * Tokenization stream API
+     * Get next token
+     * Returns null at the end of stream
+     *
+     * @return Zend_Search_Lucene_Analysis_Token|null
+     */
+    public function nextToken()
+    {
+        if ($this->_input === null) {
+            return null;
+        }
+
+        while ($this->_position < $this->_streamLength) {
+            // skip white space
+            while ($this->_position < $this->_streamLength &&
+                   !self::_isAlpha($char = $this->_nextChar())) {
+                $char = '';
+            }
+
+            $termStartPosition = $this->_position - 1;
+            $termText = $char;
+
+            // read token
+            while ($this->_position < $this->_streamLength &&
+                   self::_isAlpha($char = $this->_nextChar())) {
+                $termText .= $char;
+            }
+
+            // Empty token, end of stream.
+            if ($termText == '') {
+                return null;
+            }
+
+            $token = new Zend_Search_Lucene_Analysis_Token(
+                                      $termText,
+                                      $termStartPosition,
+                                      $this->_position - 1);
+            $token = $this->normalize($token);
+            if ($token !== null) {
+                return $token;
+            }
+            // Continue if token is skipped
+        }
+
+        return null;
+    }
+}
+

search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php

@@ -0,0 +1,169 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Analysis_Analyzer_Common */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+class Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num extends Zend_Search_Lucene_Analysis_Analyzer_Common
+{
+    /**
+     * Current char position in an UTF-8 stream
+     *
+     * @var integer
+     */
+    private $_position;
+
+    /**
+     * Current binary position in an UTF-8 stream
+     *
+     * @var integer
+     */
+    private $_bytePosition;
+
+    /**
+     * Stream length
+     *
+     * @var integer
+     */
+    private $_streamLength;
+
+    /**
+     * Reset token stream
+     */
+    public function reset()
+    {
+        $this->_position     = 0;
+        $this->_bytePosition = 0;
+
+        // convert input into UTF-8
+        if (strcasecmp($this->_encoding, 'utf8' ) != 0  &&
+            strcasecmp($this->_encoding, 'utf-8') != 0 ) {
+                $this->_input = iconv($this->_encoding, 'UTF-8', $this->_input);
+                $this->_encoding = 'UTF-8';
+        }
+
+        // Get UTF-8 string length.
+        // It also checks if it's a correct utf-8 string
+        $this->_streamLength = iconv_strlen($this->_input, 'UTF-8');
+    }
+
+    /**
+     * Check, that character is a letter
+     *
+     * @param string $char
+     * @return boolean
+     */
+    private static function _isAlNum($char)
+    {
+        if (strlen($char) > 1) {
+            // It's an UTF-8 character
+            return true;
+        }
+
+        return ctype_alnum($char);
+    }
+
+    /**
+     * Get next UTF-8 char
+     *
+     * @param string $char
+     * @return boolean
+     */
+    private function _nextChar()
+    {
+        $char = $this->_input[$this->_bytePosition++];
+
+        if (( ord($char) & 0xC0 ) == 0xC0) {
+            $addBytes = 1;
+            if (ord($char) & 0x20 ) {
+                $addBytes++;
+                if (ord($char) & 0x10 ) {
+                    $addBytes++;
+                }
+            }
+            $char .= substr($this->_input, $this->_bytePosition, $addBytes);
+            $this->_bytePosition += $addBytes;
+        }
+
+        $this->_position++;
+
+        return $char;
+    }
+
+    /**
+     * Tokenization stream API
+     * Get next token
+     * Returns null at the end of stream
+     *
+     * @return Zend_Search_Lucene_Analysis_Token|null
+     */
+    public function nextToken()
+    {
+        if ($this->_input === null) {
+            return null;
+        }
+
+        while ($this->_position < $this->_streamLength) {
+            // skip white space
+            while ($this->_position < $this->_streamLength &&
+                   !self::_isAlNum($char = $this->_nextChar())) {
+                $char = '';
+            }
+
+            $termStartPosition = $this->_position - 1;
+            $termText = $char;
+
+            // read token
+            while ($this->_position < $this->_streamLength &&
+                   self::_isAlNum($char = $this->_nextChar())) {
+                $termText .= $char;
+            }
+
+            // Empty token, end of stream.
+            if ($termText == '') {
+                return null;
+            }
+
+            $token = new Zend_Search_Lucene_Analysis_Token(
+                                      $termText,
+                                      $termStartPosition,
+                                      $this->_position - 1);
+            $token = $this->normalize($token);
+            if ($token !== null) {
+                return $token;
+            }
+            // Continue if token is skipped
+        }
+
+        return null;
+    }
+}
+

search/Zend/Search/Lucene/Analysis/Token.php

@@ -15,7 +15,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Analysis
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
@@ -24,7 +24,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Analysis
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Analysis_Token
@@ -50,13 +50,6 @@ class Zend_Search_Lucene_Analysis_Token
      */
     private $_endOffset;
 
-    /**
-     * Lexical type.
-     *
-     * @var string
-     */
-    private $_type;
-
     /**
      * The position of this token relative to the previous Token.
      *
@@ -90,12 +83,11 @@ class Zend_Search_Lucene_Analysis_Token
      * @param integer $end
      * @param string  $type
      */
-    public function __construct($text, $start, $end, $type = 'word' )
+    public function __construct($text, $start, $end)
     {
         $this->_termText    = $text;
         $this->_startOffset = $start;
         $this->_endOffset   = $end;
-        $this->_type        = $type;
 
         $this->_positionIncrement = 1;
     }
@@ -157,15 +149,5 @@ class Zend_Search_Lucene_Analysis_Token
     {
         return $this->_endOffset;
     }
-
-    /**
-     * Returns this Token's lexical type.  Defaults to 'word'.
-     *
-     * @return string
-     */
-    public function getType()
-    {
-        return $this->_type;
-    }
 }
 

search/Zend/Search/Lucene/Analysis/TokenFilter.php

@@ -15,13 +15,13 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Analysis
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Analysis_Token */
-require_once 'Zend/Search/Lucene/Analysis/Token.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Token.php';
 
 
 /**
@@ -30,7 +30,7 @@ require_once 'Zend/Search/Lucene/Analysis/Token.php';
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Analysis
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 

search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php

@@ -15,13 +15,13 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Analysis
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Analysis_TokenFilter */
-require_once 'Zend/Search/Lucene/Analysis/TokenFilter.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter.php';
 
 
 /**
@@ -30,7 +30,7 @@ require_once 'Zend/Search/Lucene/Analysis/TokenFilter.php';
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Analysis
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
@@ -44,10 +44,10 @@ class Zend_Search_Lucene_Analysis_TokenFilter_LowerCase extends Zend_Search_Luce
      */
     public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken)
     {
-        $newToken = new Zend_Search_Lucene_Analysis_Token(strtolower( $srcToken->getTermText() ),
+        $newToken = new Zend_Search_Lucene_Analysis_Token(
+                                     strtolower( $srcToken->getTermText() ),
                                      $srcToken->getStartOffset(),
-                                     $srcToken->getEndOffset(),
-                                     $srcToken->getType());
+                                     $srcToken->getEndOffset());
 
         $newToken->setPositionIncrement($srcToken->getPositionIncrement());
 

search/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php

@@ -0,0 +1,68 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Analysis_TokenFilter */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter.php';
+
+
+/**
+ * Token filter that removes short words. What is short word can be configured with constructor.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+class Zend_Search_Lucene_Analysis_TokenFilter_ShortWords extends Zend_Search_Lucene_Analysis_TokenFilter
+{
+    /**
+     * Minimum allowed term length
+     * @var integer
+     */
+    private $length;
+
+    /**
+     * Constructs new instance of this filter.
+     *
+     * @param integer $short  minimum allowed length of term which passes this filter (default 2)
+     */
+    public function __construct($length = 2) {
+        $this->length = $length;
+    }
+
+    /**
+     * Normalize Token or remove it (if null is returned)
+     *
+     * @param Zend_Search_Lucene_Analysis_Token $srcToken
+     * @return Zend_Search_Lucene_Analysis_Token
+     */
+    public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken) {
+        if (strlen($srcToken->getTermText()) < $this->length) {
+            return null;
+        } else {
+            return $srcToken;
+        }
+    }
+}
+

search/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php

@@ -0,0 +1,101 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Analysis_TokenFilter */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Exception.php';
+
+
+/**
+ * Token filter that removes stop words. These words must be provided as array (set), example:
+ * $stopwords = array('the' => 1, 'an' => '1');
+ *
+ * We do recommend to provide all words in lowercase and concatenate this class after the lowercase filter.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Analysis
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+class Zend_Search_Lucene_Analysis_TokenFilter_StopWords extends Zend_Search_Lucene_Analysis_TokenFilter
+{
+    /**
+     * Minimum allowed term length
+     * @var array
+     */
+    private $_stopSet;
+
+    /**
+     * Constructs new instance of this filter.
+     *
+     * @param array $stopwords array (set) of words that will be filtered out
+     */
+    public function __construct($stopwords = array()) {
+        $this->_stopSet = array_flip($stopwords);
+    }
+
+    /**
+     * Normalize Token or remove it (if null is returned)
+     *
+     * @param Zend_Search_Lucene_Analysis_Token $srcToken
+     * @return Zend_Search_Lucene_Analysis_Token
+     */
+    public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken) {
+        if (array_key_exists($srcToken->getTermText(), $this->_stopSet)) {
+            $t = $srcToken->getTermText();
+            return null;
+        } else {
+            return $srcToken;
+        }
+    }
+
+    /**
+     * Fills stopwords set from a text file. Each line contains one stopword, lines with '#' in the first
+     * column are ignored (as comments).
+     *
+     * You can call this method one or more times. New stopwords are always added to current set.
+     *
+     * @param string $filepath full path for text file with stopwords
+     * @throws Zend_Search_Exception When the file doesn`t exists or is not readable.
+     */
+    public function loadFromFile($filepath = null) {
+        if (! $filepath || ! file_exists($filepath)) {
+            throw new Zend_Search_Exception('You have to provide valid file path');
+        }
+        $fd = fopen($filepath, "r");
+        if (! $fd) {
+            throw new Zend_Search_Exception('Cannot open file ' . $filepath);
+        }
+        while (!feof ($fd)) {
+            $buffer = trim(fgets($fd));
+            if (strlen($buffer) > 0 && $buffer[0] != '#') {
+                $this->_stopSet[$buffer] = 1;
+            }
+        }
+        if (!fclose($fd)) {
+            throw new Zend_Search_Exception('Cannot close file ' . $filepath);
+        }
+    }
+}
+

search/Zend/Search/Lucene/Document.php

@@ -15,13 +15,13 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Document
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Field */
-require_once 'Zend/Search/Lucene/Field.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Field.php';
 
 
 /**
@@ -30,7 +30,7 @@ require_once 'Zend/Search/Lucene/Field.php';
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Document
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Document
@@ -90,9 +90,9 @@ class Zend_Search_Lucene_Document
      */
     public function getField($fieldName)
     {
-		if (!array_key_exists($fieldName, $this->_fields)) {
-			throw new Zend_Search_Lucene_Exception("Field name \"$fieldName\" not found in document.");
-		}
+        if (!array_key_exists($fieldName, $this->_fields)) {
+            throw new Zend_Search_Lucene_Exception("Field name \"$fieldName\" not found in document.");
+        }
         return $this->_fields[$fieldName];
     }
 
@@ -105,7 +105,17 @@ class Zend_Search_Lucene_Document
      */
     public function getFieldValue($fieldName)
     {
-    	return $this->getField($fieldName)->stringValue;
+    	return $this->getField($fieldName)->value;
     }
 
+    /**
+     * Returns the string value of a named field in UTF-8 encoding.
+     *
+     * @see __get()
+     * @return string
+     */
+    public function getFieldUtf8Value($fieldName)
+    {
+    	return $this->getField($fieldName)->getUtf8Value();
+    }
 }

search/Zend/Search/Lucene/Document/Html.php

@@ -0,0 +1,310 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Document
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Document */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Document.php';
+
+
+/**
+ * HTML document.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Document
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Document_Html extends Zend_Search_Lucene_Document
+{
+    /**
+     * List of document links
+     *
+     * @var array
+     */
+    private $_links = array();
+
+    /**
+     * List of document header links
+     *
+     * @var array
+     */
+    private $_headerLinks = array();
+
+    /**
+     * Stored DOM representation
+     *
+     * @var DOMDocument
+     */
+    private $_doc;
+
+    /**
+     * Object constructor
+     *
+     * @param string  $data
+     * @param boolean $isFile
+     * @param boolean $storeContent
+     */
+    private function __construct($data, $isFile, $storeContent)
+    {
+        $this->_doc = new DOMDocument();
+        $this->_doc->substituteEntities = true;
+
+        if ($isFile) {
+            @$this->_doc->loadHTMLFile($data);
+        } else{
+            @$this->_doc->loadHTML($data);
+        }
+
+        $xpath = new DOMXPath($this->_doc);
+
+        $docTitle = '';
+        $titleNodes = $xpath->query('/html/head/title');
+        foreach ($titleNodes as $titleNode) {
+            // title should always have only one entry, but we process all nodeset entries
+            $docTitle .= $titleNode->nodeValue . ' ';
+        }
+        $this->addField(Zend_Search_Lucene_Field::Text('title', $docTitle, $this->_doc->actualEncoding));
+
+        $metaNodes = $xpath->query('/html/head/meta[@name]');
+        foreach ($metaNodes as $metaNode) {
+            $this->addField(Zend_Search_Lucene_Field::Text($metaNode->getAttribute('name'),
+                                                           $metaNode->getAttribute('content'),
+                                                           $this->_doc->actualEncoding));
+        }
+
+        $docBody = '';
+        $bodyNodes = $xpath->query('/html/body');
+        foreach ($bodyNodes as $bodyNode) {
+            // body should always have only one entry, but we process all nodeset entries
+            $this->_retrieveNodeText($bodyNode, $docBody);
+        }
+        if ($storeContent) {
+            $this->addField(Zend_Search_Lucene_Field::Text('body', $docBody, $this->_doc->actualEncoding));
+        } else {
+            $this->addField(Zend_Search_Lucene_Field::UnStored('body', $docBody, $this->_doc->actualEncoding));
+        }
+
+        $linkNodes = $this->_doc->getElementsByTagName('a');
+        foreach ($linkNodes as $linkNode) {
+            if (($href = $linkNode->getAttribute('href')) != '') {
+                $this->_links[] = $href;
+            }
+        }
+        $this->_links = array_unique($this->_links);
+
+        $linkNodes = $xpath->query('/html/head/link');
+        foreach ($linkNodes as $linkNode) {
+            if (($href = $linkNode->getAttribute('href')) != '') {
+                $this->_headerLinks[] = $href;
+            }
+        }
+        $this->_headerLinks = array_unique($this->_headerLinks);
+    }
+
+    /**
+     * Get node text
+     *
+     * We should exclude scripts, which may be not included into comment tags, CDATA sections,
+     *
+     * @param DOMNode $node
+     * @param string &$text
+     */
+    private function _retrieveNodeText(DOMNode $node, &$text)
+    {
+        if ($node->nodeType == XML_TEXT_NODE) {
+            $text .= $node->nodeValue ;
+            $text .= ' ';
+        } else if ($node->nodeType == XML_ELEMENT_NODE  &&  $node->nodeName != 'script') {
+            foreach ($node->childNodes as $childNode) {
+                $this->_retrieveNodeText($childNode, $text);
+            }
+        }
+    }
+
+    /**
+     * Get document HREF links
+     *
+     * @return array
+     */
+    public function getLinks()
+    {
+        return $this->_links;
+    }
+
+    /**
+     * Get document header links
+     *
+     * @return array
+     */
+    public function getHeaderLinks()
+    {
+        return $this->_headerLinks;
+    }
+
+    /**
+     * Load HTML document from a string
+     *
+     * @param string $data
+     * @param boolean $storeContent
+     * @return Zend_Search_Lucene_Document_Html
+     */
+    public static function loadHTML($data, $storeContent = false)
+    {
+        return new Zend_Search_Lucene_Document_Html($data, false, $storeContent);
+    }
+
+    /**
+     * Load HTML document from a file
+     *
+     * @param string $file
+     * @param boolean $storeContent
+     * @return Zend_Search_Lucene_Document_Html
+     */
+    public static function loadHTMLFile($file, $storeContent = false)
+    {
+        return new Zend_Search_Lucene_Document_Html($file, true, $storeContent);
+    }
+
+
+    /**
+     * Highlight text in text node
+     *
+     * @param DOMText $node
+     * @param array   $wordsToHighlight
+     * @param string  $color
+     */
+    public function _highlightTextNode(DOMText $node, $wordsToHighlight, $color)
+    {
+        $analyzer = Zend_Search_Lucene_Analysis_Analyzer::getDefault();
+        $analyzer->setInput($node->nodeValue, $this->_doc->encoding);
+
+        $matchedTokens = array();
+
+        while (($token = $analyzer->nextToken()) !== null) {
+            if (isset($wordsToHighlight[$token->getTermText()])) {
+                $matchedTokens[] = $token;
+            }
+        }
+
+        if (count($matchedTokens) == 0) {
+            return;
+        }
+
+        $matchedTokens = array_reverse($matchedTokens);
+
+        foreach ($matchedTokens as $token) {
+            // Cut text after matched token
+            $node->splitText($token->getEndOffset());
+
+            // Cut matched node
+            $matchedWordNode = $node->splitText($token->getStartOffset());
+
+            $highlightedNode = $this->_doc->createElement('b', $matchedWordNode->nodeValue);
+            $highlightedNode->setAttribute('style', 'color:black;background-color:' . $color);
+
+            $node->parentNode->replaceChild($highlightedNode, $matchedWordNode);
+        }
+    }
+
+
+    /**
+     * highlight words in content of the specified node
+     *
+     * @param DOMNode $contextNode
+     * @param array $wordsToHighlight
+     * @param string $color
+     */
+    public function _highlightNode(DOMNode $contextNode, $wordsToHighlight, $color)
+    {
+        $textNodes = array();
+
+        if (!$contextNode->hasChildNodes()) {
+            return;
+        }
+
+        foreach ($contextNode->childNodes as $childNode) {
+            if ($childNode->nodeType == XML_TEXT_NODE) {
+                // process node later to leave childNodes structure untouched
+                $textNodes[] = $childNode;
+            } else {
+                // Skip script nodes
+                if ($childNode->nodeName != 'script') {
+                    $this->_highlightNode($childNode, $wordsToHighlight, $color);
+                }
+            }
+        }
+
+        foreach ($textNodes as $textNode) {
+            $this->_highlightTextNode($textNode, $wordsToHighlight, $color);
+        }
+    }
+
+
+
+    /**
+     * Highlight text with specified color
+     *
+     * @param string|array $words
+     * @param string $color
+     * @return string
+     */
+    public function highlight($words, $color = '#66ffff')
+    {
+        if (!is_array($words)) {
+            $words = array($words);
+        }
+        $wordsToHighlight = array();
+
+        $analyzer = Zend_Search_Lucene_Analysis_Analyzer::getDefault();
+        foreach ($words as $wordString) {
+            $wordsToHighlight = array_merge($wordsToHighlight, $analyzer->tokenize($wordString));
+        }
+
+        if (count($wordsToHighlight) == 0) {
+            return $this->_doc->saveHTML();
+        }
+
+        $wordsToHighlightFlipped = array();
+        foreach ($wordsToHighlight as $id => $token) {
+            $wordsToHighlightFlipped[$token->getTermText()] = $id;
+        }
+
+        $xpath = new DOMXPath($this->_doc);
+
+        $matchedNodes = $xpath->query("/html/body/*");
+        foreach ($matchedNodes as $matchedNode) {
+            $this->_highlightNode($matchedNode, $wordsToHighlightFlipped, $color);
+        }
+
+    }
+
+    /**
+     * Get HTML
+     *
+     * @return string
+     */
+    public function getHTML()
+    {
+        return $this->_doc->saveHTML();
+    }
+}
+

search/Zend/Search/Lucene/Exception.php

@@ -14,7 +14,7 @@
  *
  * @category   Zend
  * @package    Zend_Search_Lucene
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
@@ -22,13 +22,13 @@
 /**
  * Framework base exception
  */
-require_once 'Zend/Search/Exception.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Exception.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Exception extends Zend_Search_Exception

search/Zend/Search/Lucene/FSM.php

@@ -0,0 +1,433 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_FSMAction */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/FSMAction.php';
+
+/** Zend_Search_Exception */
+require_once $CFG->dirroot.'/search/Zend/Search/Exception.php';
+
+
+/**
+ * Abstract Finite State Machine
+ *
+ * Take a look on Wikipedia state machine description: http://en.wikipedia.org/wiki/Finite_state_machine
+ *
+ * Any type of Transducers (Moore machine or Mealy machine) also may be implemented by using this abstract FSM.
+ * process() methods invokes a specified actions which may construct FSM output.
+ * Actions may be also used to signal, that we have reached Accept State
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+abstract class Zend_Search_Lucene_FSM
+{
+    /**
+     * Machine States alphabet
+     *
+     * @var array
+     */
+    private $_states = array();
+
+    /**
+     * Current state
+     *
+     * @var integer|string
+     */
+    private $_currentState = null;
+
+    /**
+     * Input alphabet
+     *
+     * @var array
+     */
+    private $_inputAphabet = array();
+
+    /**
+     * State transition table
+     *
+     * [sourceState][input] => targetState
+     *
+     * @var array
+     */
+    private $_rules = array();
+
+    /**
+     * List of entry actions
+     * Each action executes when entering the state
+     *
+     * [state] => action
+     *
+     * @var array
+     */
+    private $_entryActions =  array();
+
+    /**
+     * List of exit actions
+     * Each action executes when exiting the state
+     *
+     * [state] => action
+     *
+     * @var array
+     */
+    private $_exitActions =  array();
+
+    /**
+     * List of input actions
+     * Each action executes when entering the state
+     *
+     * [state][input] => action
+     *
+     * @var array
+     */
+    private $_inputActions =  array();
+
+    /**
+     * List of input actions
+     * Each action executes when entering the state
+     *
+     * [state1][state2] => action
+     *
+     * @var array
+     */
+    private $_transitionActions =  array();
+
+    /**
+     * Finite State machine constructor
+     *
+     * $states is an array of integers or strings with a list of possible machine states
+     * constructor treats fist list element as a sturt state (assignes it to $_current state).
+     * It may be reassigned by setState() call.
+     * States list may be empty and can be extended later by addState() or addStates() calls.
+     *
+     * $inputAphabet is the same as $states, but represents input alphabet
+     * it also may be extended later by addInputSymbols() or addInputSymbol() calls.
+     *
+     * $rules parameter describes FSM transitions and has a structure:
+     * array( array(sourseState, input, targetState[, inputAction]),
+     *        array(sourseState, input, targetState[, inputAction]),
+     *        array(sourseState, input, targetState[, inputAction]),
+     *        ...
+     *      )
+     * Rules also can be added later by addRules() and addRule() calls.
+     *
+     * FSM actions are very flexible and may be defined by addEntryAction(), addExitAction(),
+     * addInputAction() and addTransitionAction() calls.
+     *
+     * @param array $states
+     * @param array $inputAphabet
+     * @param array $rules
+     */
+    public function __construct($states = array(), $inputAphabet = array(), $rules = array())
+    {
+        $this->addStates($states);
+        $this->addInputSymbols($inputAphabet);
+        $this->addRules($rules);
+    }
+
+    /**
+     * Add states to the state machine
+     *
+     * @param array $states
+     */
+    public function addStates($states)
+    {
+        foreach ($states as $state) {
+            $this->addState($state);
+        }
+    }
+
+    /**
+     * Add state to the state machine
+     *
+     * @param integer|string $state
+     */
+    public function addState($state)
+    {
+        $this->_states[$state] = $state;
+
+        if ($this->_currentState === null) {
+            $this->_currentState = $state;
+        }
+    }
+
+    /**
+     * Set FSM state.
+     * No any action is invoked
+     *
+     * @param integer|string $state
+     * @throws Zend_Search_Exception
+     */
+    public function setState($state)
+    {
+        if (!isset($this->_states[$state])) {
+            throw new Zend_Search_Exception('State \'' . $state . '\' is not on of the possible FSM states.');
+        }
+
+        $this->_currentState = $state;
+    }
+
+    /**
+     * Get FSM state.
+     *
+     * @return integer|string $state|null
+     */
+    public function getState()
+    {
+        return $this->_currentState;
+    }
+
+    /**
+     * Add symbols to the input alphabet
+     *
+     * @param array $inputAphabet
+     */
+    public function addInputSymbols($inputAphabet)
+    {
+        foreach ($inputAphabet as $inputSymbol) {
+            $this->addInputSymbol($inputSymbol);
+        }
+    }
+
+    /**
+     * Add symbol to the input alphabet
+     *
+     * @param integer|string $inputSymbol
+     */
+    public function addInputSymbol($inputSymbol)
+    {
+        $this->_inputAphabet[$inputSymbol] = $inputSymbol;
+    }
+
+
+    /**
+     * Add transition rules
+     *
+     * array structure:
+     * array( array(sourseState, input, targetState[, inputAction]),
+     *        array(sourseState, input, targetState[, inputAction]),
+     *        array(sourseState, input, targetState[, inputAction]),
+     *        ...
+     *      )
+     *
+     * @param array $rules
+     */
+    public function addRules($rules)
+    {
+        foreach ($rules as $rule) {
+            $this->addrule($rule[0], $rule[1], $rule[2], isset($rule[3])?$rule[3]:null);
+        }
+    }
+
+    /**
+     * Add symbol to the input alphabet
+     *
+     * @param integer|string $sourceState
+     * @param integer|string $input
+     * @param integer|string $targetState
+     * @param Zend_Search_Lucene_FSMAction|null $inputAction
+     * @throws Zend_Search_Exception
+     */
+    public function addRule($sourceState, $input, $targetState, $inputAction = null)
+    {
+        if (!isset($this->_states[$sourceState])) {
+            throw new Zend_Search_Exception('Undefined source state (' . $sourceState . ').');
+        }
+        if (!isset($this->_states[$targetState])) {
+            throw new Zend_Search_Exception('Undefined target state (' . $targetState . ').');
+        }
+        if (!isset($this->_inputAphabet[$input])) {
+            throw new Zend_Search_Exception('Undefined input symbol (' . $input . ').');
+        }
+
+        if (!isset($this->_rules[$sourceState])) {
+            $this->_rules[$sourceState] = array();
+        }
+        if (isset($this->_rules[$sourceState][$input])) {
+            throw new Zend_Search_Exception('Rule for {state,input} pair (' . $sourceState . ', '. $input . ') is already defined.');
+        }
+
+        $this->_rules[$sourceState][$input] = $targetState;
+
+
+        if ($inputAction !== null) {
+            $this->addInputAction($sourceState, $input, $inputAction);
+        }
+    }
+
+
+    /**
+     * Add state entry action.
+     * Several entry actions are allowed.
+     * Action execution order is defined by addEntryAction() calls
+     *
+     * @param integer|string $state
+     * @param Zend_Search_Lucene_FSMAction $action
+     */
+    public function addEntryAction($state, Zend_Search_Lucene_FSMAction $action)
+    {
+        if (!isset($this->_states[$state])) {
+            throw new Zend_Search_Exception('Undefined state (' . $state. ').');
+        }
+
+        if (!isset($this->_entryActions[$state])) {
+            $this->_entryActions[$state] = array();
+        }
+
+        $this->_entryActions[$state][] = $action;
+    }
+
+    /**
+     * Add state exit action.
+     * Several exit actions are allowed.
+     * Action execution order is defined by addEntryAction() calls
+     *
+     * @param integer|string $state
+     * @param Zend_Search_Lucene_FSMAction $action
+     */
+    public function addExitAction($state, Zend_Search_Lucene_FSMAction $action)
+    {
+        if (!isset($this->_states[$state])) {
+            throw new Zend_Search_Exception('Undefined state (' . $state. ').');
+        }
+
+        if (!isset($this->_exitActions[$state])) {
+            $this->_exitActions[$state] = array();
+        }
+
+        $this->_exitActions[$state][] = $action;
+    }
+
+    /**
+     * Add input action (defined by {state, input} pair).
+     * Several input actions are allowed.
+     * Action execution order is defined by addInputAction() calls
+     *
+     * @param integer|string $state
+     * @param integer|string $input
+     * @param Zend_Search_Lucene_FSMAction $action
+     */
+    public function addInputAction($state, $inputSymbol, Zend_Search_Lucene_FSMAction $action)
+    {
+        if (!isset($this->_states[$state])) {
+            throw new Zend_Search_Exception('Undefined state (' . $state. ').');
+        }
+        if (!isset($this->_inputAphabet[$inputSymbol])) {
+            throw new Zend_Search_Exception('Undefined input symbol (' . $inputSymbol. ').');
+        }
+
+        if (!isset($this->_inputActions[$state])) {
+            $this->_inputActions[$state] = array();
+        }
+        if (!isset($this->_inputActions[$state][$inputSymbol])) {
+            $this->_inputActions[$state][$inputSymbol] = array();
+        }
+
+        $this->_inputActions[$state][$inputSymbol][] = $action;
+    }
+
+    /**
+     * Add transition action (defined by {state, input} pair).
+     * Several transition actions are allowed.
+     * Action execution order is defined by addTransitionAction() calls
+     *
+     * @param integer|string $sourceState
+     * @param integer|string $targetState
+     * @param Zend_Search_Lucene_FSMAction $action
+     */
+    public function addTransitionAction($sourceState, $targetState, Zend_Search_Lucene_FSMAction $action)
+    {
+        if (!isset($this->_states[$sourceState])) {
+            throw new Zend_Search_Exception('Undefined source state (' . $sourceState. ').');
+        }
+        if (!isset($this->_states[$targetState])) {
+            throw new Zend_Search_Exception('Undefined source state (' . $targetState. ').');
+        }
+
+        if (!isset($this->_transitionActions[$sourceState])) {
+            $this->_transitionActions[$sourceState] = array();
+        }
+        if (!isset($this->_transitionActions[$sourceState][$targetState])) {
+            $this->_transitionActions[$sourceState][$targetState] = array();
+        }
+
+        $this->_transitionActions[$sourceState][$targetState][] = $action;
+    }
+
+
+    /**
+     * Process an input
+     *
+     * @param mixed $input
+     * @throws Zend_Search_Exception
+     */
+    public function process($input)
+    {
+        if (!isset($this->_rules[$this->_currentState])) {
+            throw new Zend_Search_Exception('There is no any rule for current state (' . $this->_currentState . ').');
+        }
+        if (!isset($this->_rules[$this->_currentState][$input])) {
+            throw new Zend_Search_Exception('There is no any rule for {current state, input} pair (' . $this->_currentState . ', ' . $input . ').');
+        }
+
+        $sourceState = $this->_currentState;
+        $targetState = $this->_rules[$this->_currentState][$input];
+
+        if ($sourceState != $targetState  &&  isset($this->_exitActions[$sourceState])) {
+            foreach ($this->_exitActions[$sourceState] as $action) {
+                $action->doAction();
+            }
+        }
+        if (isset($this->_inputActions[$sourceState]) &&
+            isset($this->_inputActions[$sourceState][$input])) {
+            foreach ($this->_inputActions[$sourceState][$input] as $action) {
+                $action->doAction();
+            }
+        }
+
+
+        $this->_currentState = $targetState;
+
+        if (isset($this->_transitionActions[$sourceState]) &&
+            isset($this->_transitionActions[$sourceState][$targetState])) {
+            foreach ($this->_transitionActions[$sourceState][$targetState] as $action) {
+                $action->doAction();
+            }
+        }
+        if ($sourceState != $targetState  &&  isset($this->_entryActions[$targetState])) {
+            foreach ($this->_entryActions[$targetState] as $action) {
+                $action->doAction();
+            }
+        }
+    }
+
+    public function reset()
+    {
+        if (count($this->_states) == 0) {
+            throw new Zend_Search_Exception('There is no any state defined for FSM.');
+        }
+
+        $this->_currentState = $this->_states[0];
+    }
+}
+

search/Zend/Search/Lucene/FSMAction.php

@@ -0,0 +1,65 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/**
+ * Abstract Finite State Machine
+ *
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_FSMAction
+{
+    /**
+     * Object reference
+     *
+     * @var object
+     */
+    private $_object;
+
+    /**
+     * Method name
+     *
+     * @var string
+     */
+    private $_method;
+
+    /**
+     * Object constructor
+     *
+     * @param object $object
+     * @param string $method
+     */
+    public function __construct($object, $method)
+    {
+        $this->_object = $object;
+        $this->_method = $method;
+    }
+
+    public function doAction()
+    {
+        $methodName = $this->_method;
+        $this->_object->$methodName();
+    }
+}
+

search/Zend/Search/Lucene/Field.php

@@ -15,7 +15,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Document
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
@@ -31,15 +31,20 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Document
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Field
 {
-    public $kind;
+    /**
+     * Field name
+     *
+     * @var string
+     */
+    public $name;
 
-    public $name        = 'body';
-    public $stringValue = null;
+
+    public $value;
     public $isStored    = false;
     public $isIndexed   = true;
     public $isTokenized = true;
@@ -47,26 +52,48 @@ class Zend_Search_Lucene_Field
 
     public $storeTermVector = false;
 
+    /**
+     * Field boos factor
+     * It's not stored directly in the index, but affects on normalizetion factor
+     *
+     * @var float
+     */
     public $boost = 1.0;
 
-    public function __construct($name, $stringValue, $isStored, $isIndexed, $isTokenized, $isBinary = false)
+    /**
+     * Field value encoding.
+     *
+     * @var string
+     */
+    public $encoding;
+
+    /**
+     * Object constructor
+     *
+     * @param string $name
+     * @param string $value
+     * @param string $encoding
+     * @param boolean $isStored
+     * @param boolean $isIndexed
+     * @param boolean $isTokenized
+     * @param boolean $isBinary
+     */
+    public function __construct($name, $value, $encoding, $isStored, $isIndexed, $isTokenized, $isBinary = false)
     {
-        $this->name        = $name;
+        $this->name  = $name;
+        $this->value = $value;
 
         if (!$isBinary) {
-            /**
-             * @todo Correct UTF-8 string should be required in future
-             * Until full UTF-8 support is not completed, string should be normalized to ANSII encoding
-             */
-            $this->stringValue = iconv(mb_detect_encoding($stringValue), 'ASCII//TRANSLIT', $stringValue);
-            //$this->stringValue = iconv('', 'ASCII//TRANSLIT', $stringValue);
+            $this->encoding    = $encoding;
+            $this->isTokenized = $isTokenized;
         } else {
-            $this->stringValue = $stringValue;
+            $this->encoding    = '';
+            $this->isTokenized = false;
         }
-        $this->isStored    = $isStored;
-        $this->isIndexed   = $isIndexed;
-        $this->isTokenized = $isTokenized;
-        $this->isBinary    = $isBinary;
+
+        $this->isStored  = $isStored;
+        $this->isIndexed = $isIndexed;
+        $this->isBinary  = $isBinary;
 
         $this->storeTermVector = false;
         $this->boost           = 1.0;
@@ -79,11 +106,12 @@ class Zend_Search_Lucene_Field
      *
      * @param string $name
      * @param string $value
+     * @param string $encoding
      * @return Zend_Search_Lucene_Field
      */
-    static public function Keyword($name, $value)
+    public static function Keyword($name, $value, $encoding = '')
     {
-        return new self($name, $value, true, true, false);
+        return new self($name, $value, $encoding, true, true, false);
     }
 
 
@@ -93,11 +121,12 @@ class Zend_Search_Lucene_Field
      *
      * @param string $name
      * @param string $value
+     * @param string $encoding
      * @return Zend_Search_Lucene_Field
      */
-    static public function UnIndexed($name, $value)
+    public static function UnIndexed($name, $value, $encoding = '')
     {
-        return new self($name, $value, true, false, false);
+        return new self($name, $value, $encoding, true, false, false);
     }
 
 
@@ -107,11 +136,12 @@ class Zend_Search_Lucene_Field
      *
      * @param string $name
      * @param string $value
+     * @param string $encoding
      * @return Zend_Search_Lucene_Field
      */
-    static public function Binary($name, $value)
+    public static function Binary($name, $value)
     {
-        return new self($name, $value, true, false, false, true);
+        return new self($name, $value, '', true, false, false, true);
     }
 
     /**
@@ -121,11 +151,12 @@ class Zend_Search_Lucene_Field
      *
      * @param string $name
      * @param string $value
+     * @param string $encoding
      * @return Zend_Search_Lucene_Field
      */
-    static public function Text($name, $value)
+    public static function Text($name, $value, $encoding = '')
     {
-        return new self($name, $value, true, true, true);
+        return new self($name, $value, $encoding, true, true, true);
     }
 
 
@@ -135,12 +166,27 @@ class Zend_Search_Lucene_Field
      *
      * @param string $name
      * @param string $value
+     * @param string $encoding
      * @return Zend_Search_Lucene_Field
      */
-    static public function UnStored($name, $value)
+    public static function UnStored($name, $value, $encoding = '')
     {
-        return new self($name, $value, false, true, true);
+        return new self($name, $value, $encoding, false, true, true);
     }
 
+    /**
+     * Get field value in UTF-8 encoding
+     *
+     * @return string
+     */
+    public function getUtf8Value()
+    {
+        if (strcasecmp($this->encoding, 'utf8' ) == 0  ||
+            strcasecmp($this->encoding, 'utf-8') == 0 ) {
+                return $this->value;
+        } else {
+            return iconv($this->encoding, 'UTF-8', $this->value);
+        }
+    }
 }
 

search/Zend/Search/Lucene/Index/DictionaryLoader.php

@@ -0,0 +1,254 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Index
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Exception */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+
+/**
+ * Dictionary loader
+ *
+ * It's a dummy class which is created to encapsulate non-good structured code.
+ * Manual "method inlining" is performed to increase dictionary index loading operation
+ * which is major bottelneck for search performance.
+ *
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Index
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Index_DictionaryLoader
+{
+    /**
+     * Dictionary index loader.
+     *
+     * It takes a string which is actually <segment_name>.tii index file data and
+     * returns two arrays - term and tremInfo lists.
+     *
+     * See Zend_Search_Lucene_Index_SegmintInfo class for details
+     *
+     * @param string $data
+     * @return array
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public static function load($data)
+    {
+        $termDictionary = array();
+        $termInfos      = array();
+        $pos = 0;
+
+        // $tiVersion = $tiiFile->readInt();
+        $tiVersion = ord($data[0]) << 24 | ord($data[1]) << 16 | ord($data[2]) << 8  | ord($data[3]);
+        $pos += 4;
+        if ($tiVersion != (int)0xFFFFFFFE) {
+            throw new Zend_Search_Lucene_Exception('Wrong TermInfoIndexFile file format');
+        }
+
+        // $indexTermCount = = $tiiFile->readLong();
+        if (PHP_INT_SIZE > 4) {
+            $indexTermCount = ord($data[$pos]) << 56  |
+                              ord($data[$pos+1]) << 48  |
+                              ord($data[$pos+2]) << 40  |
+                              ord($data[$pos+3]) << 32  |
+                              ord($data[$pos+4]) << 24  |
+                              ord($data[$pos+5]) << 16  |
+                              ord($data[$pos+6]) << 8   |
+                              ord($data[$pos+7]);
+        } else {
+            if ((ord($data[$pos])            != 0) ||
+                (ord($data[$pos+1])          != 0) ||
+                (ord($data[$pos+2])          != 0) ||
+                (ord($data[$pos+3])          != 0) ||
+                ((ord($data[$pos+4]) & 0x80) != 0)) {
+                     throw new Zend_Search_Lucene_Exception('Largest supported segment size (for 32-bit mode) is 2Gb');
+                 }
+
+            $indexTermCount = ord($data[$pos+4]) << 24  |
+                              ord($data[$pos+5]) << 16  |
+                              ord($data[$pos+6]) << 8   |
+                              ord($data[$pos+7]);
+        }
+        $pos += 8;
+
+        //                  $tiiFile->readInt();  // IndexInterval
+        $pos += 4;
+
+        // $skipInterval   = $tiiFile->readInt();
+        $skipInterval = ord($data[$pos]) << 24 | ord($data[$pos+1]) << 16 | ord($data[$pos+2]) << 8  | ord($data[$pos+3]);
+        $pos += 4;
+        if ($indexTermCount < 1) {
+            throw new Zend_Search_Lucene_Exception('Wrong number of terms in a term dictionary index');
+        }
+
+        $prevTerm     = '';
+        $freqPointer  =  0;
+        $proxPointer  =  0;
+        $indexPointer =  0;
+        for ($count = 0; $count < $indexTermCount; $count++) {
+            //$termPrefixLength = $tiiFile->readVInt();
+            $nbyte = ord($data[$pos++]);
+            $termPrefixLength = $nbyte & 0x7F;
+            for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) {
+                $nbyte = ord($data[$pos++]);
+                $termPrefixLength |= ($nbyte & 0x7F) << $shift;
+            }
+
+            // $termSuffix       = $tiiFile->readString();
+            $nbyte = ord($data[$pos++]);
+            $len = $nbyte & 0x7F;
+            for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) {
+                $nbyte = ord($data[$pos++]);
+                $len |= ($nbyte & 0x7F) << $shift;
+            }
+            if ($len == 0) {
+                $termSuffix = '';
+            } else {
+                $termSuffix = substr($data, $pos, $len);
+                $pos += $len;
+                for ($count1 = 0; $count1 < $len; $count1++ ) {
+                    if (( ord($termSuffix[$count1]) & 0xC0 ) == 0xC0) {
+                        $addBytes = 1;
+                        if (ord($termSuffix[$count1]) & 0x20 ) {
+                            $addBytes++;
+                        }
+                        $termSuffix .= substr($data, $pos, $addBytes);
+                        $pos += $addBytes;
+                        $len += $addBytes;
+
+                        // Check for null character. Java2 encodes null character
+                        // in two bytes.
+                        if (ord($termSuffix[$count1]) == 0xC0 &&
+                            ord($termSuffix[$count1+1]) == 0x80   ) {
+                            $termSuffix[$count1] = 0;
+                            $termSuffix = substr($termSuffix,0,$count1+1)
+                                        . substr($termSuffix,$count1+2);
+                        }
+                        $count1 += $addBytes;
+                    }
+                }
+            }
+
+            // $termValue        = Zend_Search_Lucene_Index_Term::getPrefix($prevTerm, $termPrefixLength) . $termSuffix;
+            $pb = 0; $pc = 0;
+            while ($pb < strlen($prevTerm)  &&  $pc < $termPrefixLength) {
+                $charBytes = 1;
+                if ((ord($prevTerm[$pb]) & 0xC0) == 0xC0) {
+                    $charBytes++;
+                    if (ord($prevTerm[$pb]) & 0x20 ) {
+                        $charBytes++;
+                        if (ord($prevTerm[$pb]) & 0x10 ) {
+                            $charBytes++;
+                        }
+                    }
+                }
+
+                if ($pb + $charBytes > strlen($data)) {
+                    // wrong character
+                    break;
+                }
+
+                $pc++;
+                $pb += $charBytes;
+            }
+            $termValue = substr($prevTerm, 0, $pb) . $termSuffix;
+
+            // $termFieldNum     = $tiiFile->readVInt();
+            $nbyte = ord($data[$pos++]);
+            $termFieldNum = $nbyte & 0x7F;
+            for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) {
+                $nbyte = ord($data[$pos++]);
+                $termFieldNum |= ($nbyte & 0x7F) << $shift;
+            }
+
+            // $docFreq          = $tiiFile->readVInt();
+            $nbyte = ord($data[$pos++]);
+            $docFreq = $nbyte & 0x7F;
+            for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) {
+                $nbyte = ord($data[$pos++]);
+                $docFreq |= ($nbyte & 0x7F) << $shift;
+            }
+
+            // $freqPointer     += $tiiFile->readVInt();
+            $nbyte = ord($data[$pos++]);
+            $vint = $nbyte & 0x7F;
+            for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) {
+                $nbyte = ord($data[$pos++]);
+                $vint |= ($nbyte & 0x7F) << $shift;
+            }
+            $freqPointer += $vint;
+
+            // $proxPointer     += $tiiFile->readVInt();
+            $nbyte = ord($data[$pos++]);
+            $vint = $nbyte & 0x7F;
+            for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) {
+                $nbyte = ord($data[$pos++]);
+                $vint |= ($nbyte & 0x7F) << $shift;
+            }
+            $proxPointer += $vint;
+
+            if( $docFreq >= $skipInterval ) {
+                // $skipDelta = $tiiFile->readVInt();
+                $nbyte = ord($data[$pos++]);
+                $vint = $nbyte & 0x7F;
+                for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) {
+                    $nbyte = ord($data[$pos++]);
+                    $vint |= ($nbyte & 0x7F) << $shift;
+                }
+                $skipDelta = $vint;
+            } else {
+                $skipDelta = 0;
+            }
+
+            // $indexPointer += $tiiFile->readVInt();
+            $nbyte = ord($data[$pos++]);
+            $vint = $nbyte & 0x7F;
+            for ($shift=7; ($nbyte & 0x80) != 0; $shift += 7) {
+                $nbyte = ord($data[$pos++]);
+                $vint |= ($nbyte & 0x7F) << $shift;
+            }
+            $indexPointer += $vint;
+
+
+            // $this->_termDictionary[] =  new Zend_Search_Lucene_Index_Term($termValue, $termFieldNum);
+            $termDictionary[] = array($termFieldNum, $termValue);
+
+            $termInfos[] =
+                 // new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipDelta, $indexPointer);
+                 array($docFreq, $freqPointer, $proxPointer, $skipDelta, $indexPointer);
+
+            $prevTerm = $termValue;
+        }
+
+        // Check special index entry mark
+        if ($termDictionary[0][0] != (int)0xFFFFFFFF) {
+            throw new Zend_Search_Lucene_Exception('Wrong TermInfoIndexFile file format');
+        } else if (PHP_INT_SIZE > 4){
+            // Treat 64-bit 0xFFFFFFFF as -1
+            $termDictionary[0][0] = -1;
+        }
+
+        return array(&$termDictionary, &$termInfos);
+    }
+}
+

search/Zend/Search/Lucene/Index/FieldInfo.php

@@ -15,7 +15,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Index
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
@@ -24,7 +24,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Index
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Index_FieldInfo

search/Zend/Search/Lucene/Index/SegmentInfo.php

@@ -15,20 +15,23 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Index
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
+/** Zend_Search_Lucene_Index_DictionaryLoader */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/DictionaryLoader.php';
+
 
 /** Zend_Search_Lucene_Exception */
-require_once 'Zend/Search/Lucene/Exception.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Index
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Index_SegmentInfo
@@ -49,7 +52,12 @@ class Zend_Search_Lucene_Index_SegmentInfo
 
     /**
      * Term Dictionary Index
-     * Array of the Zend_Search_Lucene_Index_Term objects
+     *
+     * Array of arrays (Zend_Search_Lucene_Index_Term objects are represented as arrays because
+     * of performance considerations)
+     * [0] -> $termValue
+     * [1] -> $termFieldNum
+     *
      * Corresponding Zend_Search_Lucene_Index_TermInfo object stored in the $_termDictionaryInfos
      *
      * @var array
@@ -58,7 +66,14 @@ class Zend_Search_Lucene_Index_SegmentInfo
 
     /**
      * Term Dictionary Index TermInfos
-     * Array of the Zend_Search_Lucene_Index_TermInfo objects
+     *
+     * Array of arrays (Zend_Search_Lucene_Index_TermInfo objects are represented as arrays because
+     * of performance considerations)
+     * [0] -> $docFreq
+     * [1] -> $freqPointer
+     * [2] -> $proxPointer
+     * [3] -> $skipOffset
+     * [4] -> $indexPointer
      *
      * @var array
      */
@@ -88,6 +103,14 @@ class Zend_Search_Lucene_Index_SegmentInfo
      */
     private $_segFiles;
 
+    /**
+     * Associative array where the key is the file name and the value is file size (.csf).
+     *
+     * @var array
+     */
+    private $_segFileSizes;
+
+
     /**
      * File system adapter.
      *
@@ -122,6 +145,7 @@ class Zend_Search_Lucene_Index_SegmentInfo
      */
     private $_deletedDirty = false;
 
+
     /**
      * Zend_Search_Lucene_Index_SegmentInfo constructor needs Segmentname,
      * Documents count and Directory as a parameter.
@@ -144,9 +168,15 @@ class Zend_Search_Lucene_Index_SegmentInfo
 
             for ($count = 0; $count < $segFilesCount; $count++) {
                 $dataOffset = $cfsFile->readLong();
+                if ($count != 0) {
+                    $this->_segFileSizes[$fileName] = $dataOffset - end($this->_segFiles);
+                }
                 $fileName = $cfsFile->readString();
                 $this->_segFiles[$fileName] = $dataOffset;
             }
+            if ($count != 0) {
+                $this->_segFileSizes[$fileName] = $this->_directory->fileLength($name . '.cfs') - $dataOffset;
+            }
         }
 
         $fnmFile = $this->openCompoundFile('.fnm');
@@ -197,7 +227,6 @@ class Zend_Search_Lucene_Index_SegmentInfo
                         }
                     }
                 }
-
             }
         } catch(Zend_Search_Exception $e) {
             if (strpos($e->getMessage(), 'compound file doesn\'t contain') !== false ) {
@@ -212,16 +241,17 @@ class Zend_Search_Lucene_Index_SegmentInfo
      * Opens index file stoted within compound index file
      *
      * @param string $extension
+     * @param boolean $shareHandler
      * @throws Zend_Search_Lucene_Exception
      * @return Zend_Search_Lucene_Storage_File
      */
-    public function openCompoundFile($extension)
+    public function openCompoundFile($extension, $shareHandler = true)
     {
         $filename = $this->_name . $extension;
 
         // Try to open common file first
         if ($this->_directory->fileExists($filename)) {
-            return $this->_directory->getFileObject($filename);
+            return $this->_directory->getFileObject($filename, $shareHandler);
         }
 
         if( !isset($this->_segFiles[$filename]) ) {
@@ -229,11 +259,34 @@ class Zend_Search_Lucene_Index_SegmentInfo
                                        . $filename . ' file.' );
         }
 
-        $file = $this->_directory->getFileObject( $this->_name.".cfs" );
+        $file = $this->_directory->getFileObject($this->_name . '.cfs', $shareHandler);
         $file->seek($this->_segFiles[$filename]);
         return $file;
     }
 
+    /**
+     * Get compound file length
+     *
+     * @param string $extension
+     * @return integer
+     */
+    public function compoundFileLength($extension)
+    {
+        $filename = $this->_name . $extension;
+
+        // Try to get common file first
+        if ($this->_directory->fileExists($filename)) {
+            return $this->_directory->fileLength($filename);
+        }
+
+        if( !isset($this->_segFileSizes[$filename]) ) {
+            throw new Zend_Search_Lucene_Exception('Index compound file doesn\'t contain '
+                                       . $filename . ' file.' );
+        }
+
+        return $this->_segFileSizes[$filename];
+    }
+
     /**
      * Returns field index or -1 if field is not found
      *
@@ -255,7 +308,7 @@ class Zend_Search_Lucene_Index_SegmentInfo
      * Returns field info for specified field
      *
      * @param integer $fieldNum
-     * @return ZSearchFieldInfo
+     * @return Zend_Search_Lucene_Index_FieldInfo
      */
     public function getField($fieldNum)
     {
@@ -281,7 +334,17 @@ class Zend_Search_Lucene_Index_SegmentInfo
     }
 
     /**
-     * Returns the total number of documents in this segment.
+     * Returns array of FieldInfo objects.
+     *
+     * @return array
+     */
+    public function getFieldInfos()
+    {
+        return $this->_fields;
+    }
+
+    /**
+     * Returns the total number of documents in this segment (including deleted documents).
      *
      * @return integer
      */
@@ -290,6 +353,38 @@ class Zend_Search_Lucene_Index_SegmentInfo
         return $this->_docCount;
     }
 
+    /**
+     * Returns number of deleted documents.
+     *
+     * @return integer
+     */
+    private function _deletedCount()
+    {
+        if ($this->_deleted === null) {
+            return 0;
+        }
+
+        if (extension_loaded('bitset')) {
+            return count(bitset_to_array($this->_deleted));
+        } else {
+            return count($this->_deleted);
+        }
+    }
+
+    /**
+     * Returns the total number of non-deleted documents in this segment.
+     *
+     * @return integer
+     */
+    public function numDocs()
+    {
+        if ($this->hasDeletions()) {
+            return $this->_docCount - $this->_deletedCount();
+        } else {
+            return $this->_docCount;
+        }
+    }
+
     /**
      * Get field position in a fields dictionary
      *
@@ -302,57 +397,6 @@ class Zend_Search_Lucene_Index_SegmentInfo
                            $this->_fieldsDicPositions[$fieldNum] : $fieldNum;
     }
 
-    /**
-     * Loads Term dictionary from TermInfoIndex file
-     */
-    protected function _loadDictionary()
-    {
-        if ($this->_termDictionary !== null) {
-            return;
-        }
-
-        $this->_termDictionary = array();
-        $this->_termDictionaryInfos = array();
-
-        $tiiFile = $this->openCompoundFile('.tii');
-        $tiVersion = $tiiFile->readInt();
-        if ($tiVersion != (int)0xFFFFFFFE) {
-            throw new Zend_Search_Lucene_Exception('Wrong TermInfoIndexFile file format');
-        }
-
-        $indexTermCount = $tiiFile->readLong();
-                          $tiiFile->readInt();  // IndexInterval
-        $skipInterval   = $tiiFile->readInt();
-
-        $prevTerm     = '';
-        $freqPointer  =  0;
-        $proxPointer  =  0;
-        $indexPointer =  0;
-        for ($count = 0; $count < $indexTermCount; $count++) {
-            $termPrefixLength = $tiiFile->readVInt();
-            $termSuffix       = $tiiFile->readString();
-            $termValue        = substr( $prevTerm, 0, $termPrefixLength ) . $termSuffix;
-
-            $termFieldNum     = $tiiFile->readVInt();
-            $docFreq          = $tiiFile->readVInt();
-            $freqPointer     += $tiiFile->readVInt();
-            $proxPointer     += $tiiFile->readVInt();
-            if( $docFreq >= $skipInterval ) {
-                $skipDelta = $tiiFile->readVInt();
-            } else {
-                $skipDelta = 0;
-            }
-
-            $indexPointer += $tiiFile->readVInt();
-
-            $this->_termDictionary[] =  new Zend_Search_Lucene_Index_Term($termValue,$termFieldNum);
-            $this->_termDictionaryInfos[] =
-                new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipDelta, $indexPointer);
-            $prevTerm = $termValue;
-        }
-    }
-
-
     /**
      * Return segment name
      *
@@ -364,15 +408,75 @@ class Zend_Search_Lucene_Index_SegmentInfo
     }
 
 
+    /**
+     * TermInfo cache
+     *
+     * Size is 1024.
+     * Numbers are used instead of class constants because of performance considerations
+     *
+     * @var array
+     */
+    private $_termInfoCache = array();
+
+    private function _cleanUpTermInfoCache()
+    {
+        // Clean 256 term infos
+        foreach ($this->_termInfoCache as $key => $termInfo) {
+            unset($this->_termInfoCache[$key]);
+
+            // leave 768 last used term infos
+            if (count($this->_termInfoCache) == 768) {
+                break;
+            }
+        }
+    }
+
     /**
      * Scans terms dictionary and returns term info
      *
      * @param Zend_Search_Lucene_Index_Term $term
      * @return Zend_Search_Lucene_Index_TermInfo
      */
-    public function getTermInfo($term)
+    public function getTermInfo(Zend_Search_Lucene_Index_Term $term)
     {
-        $this->_loadDictionary();
+        $termKey = $term->key();
+        if (isset($this->_termInfoCache[$termKey])) {
+            $termInfo = $this->_termInfoCache[$termKey];
+
+            // Move termInfo to the end of cache
+            unset($this->_termInfoCache[$termKey]);
+            $this->_termInfoCache[$termKey] = $termInfo;
+
+            return $termInfo;
+        }
+
+
+        if ($this->_termDictionary === null) {
+            // Check, if index is already serialized
+            if ($this->_directory->fileExists($this->_name . '.sti')) {
+                // Prefetch dictionary index data
+                $stiFile = $this->_directory->getFileObject($this->_name . '.sti');
+                $stiFileData = $stiFile->readBytes($this->_directory->fileLength($this->_name . '.sti'));
+
+                // Load dictionary index data
+                list($this->_termDictionary, $this->_termDictionaryInfos) = unserialize($stiFileData);
+            } else {
+                // Prefetch dictionary index data
+                $tiiFile = $this->openCompoundFile('.tii');
+                $tiiFileData = $tiiFile->readBytes($this->compoundFileLength('.tii'));
+
+                // Load dictionary index data
+                list($this->_termDictionary, $this->_termDictionaryInfos) =
+                            Zend_Search_Lucene_Index_DictionaryLoader::load($tiiFileData);
+
+                $stiFileData = serialize(array($this->_termDictionary, $this->_termDictionaryInfos));
+                $stiFile = $this->_directory->createFile($this->_name . '.sti');
+                $stiFile->writeBytes($stiFileData);
+            }
+
+        }
+
+
 
         $searchField = $this->getFieldNum($term->field);
 
@@ -389,10 +493,10 @@ class Zend_Search_Lucene_Index_SegmentInfo
             $mid = ($highIndex + $lowIndex) >> 1;
             $midTerm = $this->_termDictionary[$mid];
 
-            $fieldNum = $this->_getFieldPosition($midTerm->field);
+            $fieldNum = $this->_getFieldPosition($midTerm[0] /* field */);
             $delta = $searchDicField - $fieldNum;
             if ($delta == 0) {
-                $delta = strcmp($term->text, $midTerm->text);
+                $delta = strcmp($term->text, $midTerm[1] /* text */);
             }
 
             if ($delta < 0) {
@@ -400,7 +504,14 @@ class Zend_Search_Lucene_Index_SegmentInfo
             } elseif ($delta > 0) {
                 $lowIndex  = $mid+1;
             } else {
-                return $this->_termDictionaryInfos[$mid]; // We got it!
+                // return $this->_termDictionaryInfos[$mid]; // We got it!
+                $a = $this->_termDictionaryInfos[$mid];
+                $termInfo = new Zend_Search_Lucene_Index_TermInfo($a[0], $a[1], $a[2], $a[3], $a[4]);
+
+                // Put loaded termInfo into cache
+                $this->_termInfoCache[$termKey] = $termInfo;
+
+                return $termInfo;
             }
         }
 
@@ -411,7 +522,7 @@ class Zend_Search_Lucene_Index_SegmentInfo
 
         $prevPosition = $highIndex;
         $prevTerm = $this->_termDictionary[$prevPosition];
-        $prevTermInfo = $this->_termDictionaryInfos[ $prevPosition ];
+        $prevTermInfo = $this->_termDictionaryInfos[$prevPosition];
 
         $tisFile = $this->openCompoundFile('.tis');
         $tiVersion = $tisFile->readInt();
@@ -423,12 +534,12 @@ class Zend_Search_Lucene_Index_SegmentInfo
         $indexInterval = $tisFile->readInt();
         $skipInterval  = $tisFile->readInt();
 
-        $tisFile->seek($prevTermInfo->indexPointer - 20 /* header size*/, SEEK_CUR);
+        $tisFile->seek($prevTermInfo[4] /* indexPointer */ - 20 /* header size*/, SEEK_CUR);
 
-        $termValue    = $prevTerm->text;
-        $termFieldNum = $prevTerm->field;
-        $freqPointer = $prevTermInfo->freqPointer;
-        $proxPointer = $prevTermInfo->proxPointer;
+        $termValue    = $prevTerm[1] /* text */;
+        $termFieldNum = $prevTerm[0] /* field */;
+        $freqPointer = $prevTermInfo[1] /* freqPointer */;
+        $proxPointer = $prevTermInfo[2] /* proxPointer */;
         for ($count = $prevPosition*$indexInterval + 1;
              $count <= $termCount &&
              ( $this->_getFieldPosition($termFieldNum) < $searchDicField ||
@@ -438,7 +549,7 @@ class Zend_Search_Lucene_Index_SegmentInfo
             $termPrefixLength = $tisFile->readVInt();
             $termSuffix       = $tisFile->readString();
             $termFieldNum     = $tisFile->readVInt();
-            $termValue        = substr( $termValue, 0, $termPrefixLength ) . $termSuffix;
+            $termValue        = Zend_Search_Lucene_Index_Term::getPrefix($termValue, $termPrefixLength) . $termSuffix;
 
             $docFreq      = $tisFile->readVInt();
             $freqPointer += $tisFile->readVInt();
@@ -451,10 +562,115 @@ class Zend_Search_Lucene_Index_SegmentInfo
         }
 
         if ($termFieldNum == $searchField && $termValue == $term->text) {
-            return new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipOffset);
+            $termInfo = new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipOffset);
         } else {
-            return null;
+            $termInfo = null;
         }
+
+        // Put loaded termInfo into cache
+        $this->_termInfoCache[$termKey] = $termInfo;
+
+        if (count($this->_termInfoCache) == 1024) {
+            $this->_cleanUpTermInfoCache();
+        }
+
+        return $termInfo;
+    }
+
+    /**
+     * Returns term freqs array.
+     * Result array structure: array(docId => freq, ...)
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @param integer $shift
+     * @return Zend_Search_Lucene_Index_TermInfo
+     */
+    public function termFreqs(Zend_Search_Lucene_Index_Term $term, $shift = 0)
+    {
+        $termInfo = $this->getTermInfo($term);
+
+        if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) {
+            return array();
+        }
+
+        $frqFile = $this->openCompoundFile('.frq');
+        $frqFile->seek($termInfo->freqPointer,SEEK_CUR);
+        $result = array();
+        $docId = 0;
+
+        for ($count = 0; $count < $termInfo->docFreq; $count++) {
+            $docDelta = $frqFile->readVInt();
+            if ($docDelta % 2 == 1) {
+                $docId += ($docDelta-1)/2;
+                $result[$shift + $docId] = 1;
+            } else {
+                $docId += $docDelta/2;
+                $result[$shift + $docId] = $frqFile->readVInt();
+            }
+        }
+
+        return $result;
+    }
+
+    /**
+     * Returns term positions array.
+     * Result array structure: array(docId => array(pos1, pos2, ...), ...)
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @param integer $shift
+     * @return Zend_Search_Lucene_Index_TermInfo
+     */
+    public function termPositions(Zend_Search_Lucene_Index_Term $term, $shift = 0)
+    {
+        $termInfo = $this->getTermInfo($term);
+
+        if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) {
+            return array();
+        }
+
+        $frqFile = $this->openCompoundFile('.frq');
+        $frqFile->seek($termInfo->freqPointer,SEEK_CUR);
+        $freqs = array();
+        $docId = 0;
+
+        for ($count = 0; $count < $termInfo->docFreq; $count++) {
+            $docDelta = $frqFile->readVInt();
+            if ($docDelta % 2 == 1) {
+                $docId += ($docDelta-1)/2;
+                $freqs[$docId] = 1;
+            } else {
+                $docId += $docDelta/2;
+                $freqs[$docId] = $frqFile->readVInt();
+            }
+        }
+
+        $result = array();
+        $prxFile = $this->openCompoundFile('.prx');
+        $prxFile->seek($termInfo->proxPointer, SEEK_CUR);
+        foreach ($freqs as $docId => $freq) {
+            $termPosition = 0;
+            $positions = array();
+
+            for ($count = 0; $count < $freq; $count++ ) {
+                $termPosition += $prxFile->readVInt();
+                $positions[] = $termPosition;
+            }
+
+            $result[$shift + $docId] = $positions;
+        }
+
+        return $result;
+    }
+
+    /**
+     * Load normalizatin factors from an index file
+     *
+     * @param integer $fieldNum
+     */
+    private function _loadNorm($fieldNum)
+    {
+        $fFile = $this->openCompoundFile('.f' . $fieldNum);
+        $this->_norms[$fieldNum] = $fFile->readBytes($this->_docCount);
     }
 
     /**
@@ -462,7 +678,7 @@ class Zend_Search_Lucene_Index_SegmentInfo
      *
      * @param integer $id
      * @param string $fieldName
-     * @return string
+     * @return float
      */
     public function norm($id, $fieldName)
     {
@@ -472,14 +688,37 @@ class Zend_Search_Lucene_Index_SegmentInfo
             return null;
         }
 
-        if ( !isset( $this->_norms[$fieldNum] )) {
-            $fFile = $this->openCompoundFile('.f' . $fieldNum);
-            $this->_norms[$fieldNum] = $fFile->readBytes($this->_docCount);
+        if (!isset($this->_norms[$fieldNum])) {
+            $this->_loadNorm($fieldNum);
         }
 
         return Zend_Search_Lucene_Search_Similarity::decodeNorm( ord($this->_norms[$fieldNum]{$id}) );
     }
 
+    /**
+     * Returns norm vector, encoded in a byte string
+     *
+     * @param string $fieldName
+     * @return string
+     */
+    public function normVector($fieldName)
+    {
+        $fieldNum = $this->getFieldNum($fieldName);
+
+        if ($fieldNum == -1  ||  !($this->_fields[$fieldNum]->isIndexed)) {
+            $similarity = Zend_Search_Lucene_Search_Similarity::getDefault();
+
+            return str_repeat(chr($similarity->encodeNorm( $similarity->lengthNorm($fieldName, 0) )),
+                              $this->_docCount);
+        }
+
+        if (!isset($this->_norms[$fieldNum])) {
+            $this->_loadNorm($fieldNum);
+        }
+
+        return $this->_norms[$fieldNum];
+    }
+
 
     /**
      * Returns true if any documents have been deleted from this index segment.
@@ -571,5 +810,247 @@ class Zend_Search_Lucene_Index_SegmentInfo
 
         $this->_deletedDirty = false;
     }
+
+
+
+    /**
+     * Term Dictionary File object for stream like terms reading
+     *
+     * @var Zend_Search_Lucene_Storage_File
+     */
+    private $_tisFile = null;
+
+    /**
+     * Frequencies File object for stream like terms reading
+     *
+     * @var Zend_Search_Lucene_Storage_File
+     */
+    private $_frqFile = null;
+
+    /**
+     * Offset of the .frq file in the compound file
+     *
+     * @var integer
+     */
+    private $_frqFileOffset;
+
+    /**
+     * Positions File object for stream like terms reading
+     *
+     * @var Zend_Search_Lucene_Storage_File
+     */
+    private $_prxFile = null;
+
+    /**
+     * Offset of the .prx file in the compound file
+     *
+     * @var integer
+     */
+    private $_prxFileOffset;
+
+
+    /**
+     * Number of terms in term stream
+     *
+     * @var integer
+     */
+    private $_termCount = 0;
+
+    /**
+     * Segment skip interval
+     *
+     * @var integer
+     */
+    private $_skipInterval;
+
+    /**
+     * Last TermInfo in a terms stream
+     *
+     * @var Zend_Search_Lucene_Index_TermInfo
+     */
+    private $_lastTermInfo = null;
+
+    /**
+     * Last Term in a terms stream
+     *
+     * @var Zend_Search_Lucene_Index_Term
+     */
+    private $_lastTerm = null;
+
+    /**
+     * Map of the document IDs
+     * Used to get new docID after removing deleted documents.
+     * It's not very effective from memory usage point of view,
+     * but much more faster, then other methods
+     *
+     * @var array|null
+     */
+    private $_docMap = null;
+
+    /**
+     * An array of all term positions in the documents.
+     * Array structure: array( docId => array( pos1, pos2, ...), ...)
+     *
+     * @var array
+     */
+    private $_lastTermPositions;
+
+    /**
+     * Reset terms stream
+     *
+     * $startId - id for the fist document
+     * $compact - remove deleted documents
+     *
+     * Returns start document id for the next segment
+     *
+     * @param integer $startId
+     * @param boolean $compact
+     * @throws Zend_Search_Lucene_Exception
+     * @return integer
+     */
+    public function reset($startId = 0, $compact = false)
+    {
+        if ($this->_tisFile !== null) {
+            $this->_tisFile = null;
+        }
+
+        $this->_tisFile = $this->openCompoundFile('.tis', false);
+        $tiVersion = $this->_tisFile->readInt();
+        if ($tiVersion != (int)0xFFFFFFFE) {
+            throw new Zend_Search_Lucene_Exception('Wrong TermInfoFile file format');
+        }
+
+        $this->_termCount    = $this->_tisFile->readLong();
+                               $this->_tisFile->readInt();  // Read Index interval
+        $this->_skipInterval = $this->_tisFile->readInt();  // Read skip interval
+
+        if ($this->_frqFile !== null) {
+            $this->_frqFile = null;
+        }
+        $this->_frqFile = $this->openCompoundFile('.frq', false);
+        $this->_frqFileOffset = $this->_frqFile->tell();
+
+        if ($this->_prxFile !== null) {
+            $this->_prxFile = null;
+        }
+        $this->_prxFile = $this->openCompoundFile('.prx', false);
+        $this->_prxFileOffset = $this->_prxFile->tell();
+
+        $this->_lastTerm     = new Zend_Search_Lucene_Index_Term('', -1);
+        $this->_lastTermInfo = new Zend_Search_Lucene_Index_TermInfo(0, 0, 0, 0);
+
+        $this->_docMap = array();
+        for ($count = 0; $count < $this->_docCount; $count++) {
+            if (!$this->isDeleted($count)) {
+                $this->_docMap[$count] = $startId + ($compact ? count($this->_docMap) : $count);
+            }
+        }
+
+        $this->nextTerm();
+        return $startId + ($compact ? count($this->_docMap) : $this->_docCount);
+    }
+
+
+    /**
+     * Scans terms dictionary and returns next term
+     *
+     * @return Zend_Search_Lucene_Index_Term|null
+     */
+    public function nextTerm()
+    {
+        if ($this->_tisFile === null  ||  $this->_termCount == 0) {
+            $this->_lastTerm     = null;
+            $this->_lastTermInfo = null;
+
+            // may be necessary for "empty" segment
+            $this->_tisFile = null;
+            $this->_frqFile = null;
+            $this->_prxFile = null;
+
+            return null;
+        }
+
+        $termPrefixLength = $this->_tisFile->readVInt();
+        $termSuffix       = $this->_tisFile->readString();
+        $termFieldNum     = $this->_tisFile->readVInt();
+        $termValue        = Zend_Search_Lucene_Index_Term::getPrefix($this->_lastTerm->text, $termPrefixLength) . $termSuffix;
+
+        $this->_lastTerm = new Zend_Search_Lucene_Index_Term($termValue, $this->_fields[$termFieldNum]->name);
+
+        $docFreq     = $this->_tisFile->readVInt();
+        $freqPointer = $this->_lastTermInfo->freqPointer + $this->_tisFile->readVInt();
+        $proxPointer = $this->_lastTermInfo->proxPointer + $this->_tisFile->readVInt();
+        if ($docFreq >= $this->_skipInterval) {
+            $skipOffset = $this->_tisFile->readVInt();
+        } else {
+            $skipOffset = 0;
+        }
+
+        $this->_lastTermInfo = new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipOffset);
+
+
+        $this->_lastTermPositions = array();
+
+        $this->_frqFile->seek($this->_lastTermInfo->freqPointer + $this->_frqFileOffset, SEEK_SET);
+        $freqs = array();   $docId = 0;
+        for( $count = 0; $count < $this->_lastTermInfo->docFreq; $count++ ) {
+            $docDelta = $this->_frqFile->readVInt();
+            if( $docDelta % 2 == 1 ) {
+                $docId += ($docDelta-1)/2;
+                $freqs[ $docId ] = 1;
+            } else {
+                $docId += $docDelta/2;
+                $freqs[ $docId ] = $this->_frqFile->readVInt();
+            }
+        }
+
+        $this->_prxFile->seek($this->_lastTermInfo->proxPointer + $this->_prxFileOffset, SEEK_SET);
+        foreach ($freqs as $docId => $freq) {
+            $termPosition = 0;  $positions = array();
+
+            for ($count = 0; $count < $freq; $count++ ) {
+                $termPosition += $this->_prxFile->readVInt();
+                $positions[] = $termPosition;
+            }
+
+            if (isset($this->_docMap[$docId])) {
+                $this->_lastTermPositions[$this->_docMap[$docId]] = $positions;
+            }
+        }
+
+
+        $this->_termCount--;
+        if ($this->_termCount == 0) {
+            $this->_tisFile = null;
+            $this->_frqFile = null;
+            $this->_prxFile = null;
+        }
+
+        return $this->_lastTerm;
+    }
+
+
+    /**
+     * Returns term in current position
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @return Zend_Search_Lucene_Index_Term|null
+     */
+    public function currentTerm()
+    {
+        return $this->_lastTerm;
+    }
+
+
+    /**
+     * Returns an array of all term positions in the documents.
+     * Return array structure: array( docId => array( pos1, pos2, ...), ...)
+     *
+     * @return array
+     */
+    public function currentTermPositions()
+    {
+        return $this->_lastTermPositions;
+    }
 }
 

search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php

@@ -0,0 +1,53 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Index
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Exception */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+/** Zend_Search_Lucene */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/PriorityQueue.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Index
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Index_SegmentInfoPriorityQueue extends Zend_Search_Lucene_PriorityQueue
+{
+    /**
+     * Compare elements
+     *
+     * Returns true, if $el1 is less than $el2; else otherwise
+     *
+     * @param mixed $segmentInfo1
+     * @param mixed $segmentInfo2
+     * @return boolean
+     */
+    protected function _less($segmentInfo1, $segmentInfo2)
+    {
+        return strcmp($segmentInfo1->currentTerm()->key(), $segmentInfo2->currentTerm()->key()) < 0;
+    }
+
+}

search/Zend/Search/Lucene/Index/SegmentMerger.php

@@ -0,0 +1,273 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Index
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Exception */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+/** Zend_Search_Lucene_Index_SegmentInfo */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php';
+
+/** Zend_Search_Lucene_Index_SegmentWriter_StreamWriter */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentWriter/StreamWriter.php';
+
+/** Zend_Search_Lucene_Index_SegmentInfoPriorityQueue */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Index
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Index_SegmentMerger
+{
+    /**
+     * Target segment writer
+     *
+     * @var Zend_Search_Lucene_Index_SegmentWriter_StreamWriter
+     */
+    private $_writer;
+
+    /**
+     * Number of docs in a new segment
+     *
+     * @var integer
+     */
+    private $_docCount;
+
+    /**
+     * A set of segments to be merged
+     *
+     * @var array Zend_Search_Lucene_Index_SegmentInfo
+     */
+    private $_segmentInfos = array();
+
+    /**
+     * Flag to signal, that merge is already done
+     *
+     * @var boolean
+     */
+    private $_mergeDone = false;
+
+    /**
+     * Field map
+     * [<segment_name>][<field_number>] => <target_field_number>
+     *
+     * @var array
+     */
+    private $_fieldsMap = array();
+
+
+
+    /**
+     * Object constructor.
+     *
+     * Creates new segment merger with $directory as target to merge segments into
+     * and $name as a name of new segment
+     *
+     * @param Zend_Search_Lucene_Storage_Directory $directory
+     * @param string $name
+     */
+    public function __construct($directory, $name)
+    {
+        $this->_writer = new Zend_Search_Lucene_Index_SegmentWriter_StreamWriter($directory, $name);
+    }
+
+
+    /**
+     * Add segmnet to a collection of segments to be merged
+     *
+     * @param Zend_Search_Lucene_Index_SegmentInfo $segment
+     */
+    public function addSource(Zend_Search_Lucene_Index_SegmentInfo $segmentInfo)
+    {
+        $this->_segmentInfos[$segmentInfo->getName()] = $segmentInfo;
+    }
+
+
+    /**
+     * Do merge.
+     *
+     * Returns number of documents in newly created segment
+     *
+     * @return Zend_Search_Lucene_Index_SegmentInfo
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public function merge()
+    {
+        if ($this->_mergeDone) {
+            throw new Zend_Search_Lucene_Exception('Merge is already done.');
+        }
+
+        if (count($this->_segmentInfos) < 1) {
+            throw new Zend_Search_Lucene_Exception('Wrong number of segments to be merged ('
+                                                 . count($this->_segmentInfos)
+                                                 . ').');
+        }
+
+        $this->_mergeFields();
+        $this->_mergeNorms();
+        $this->_mergeStoredFields();
+        $this->_mergeTerms();
+
+        $this->_mergeDone = true;
+
+        return $this->_writer->close();
+    }
+
+
+    /**
+     * Merge fields information
+     */
+    private function _mergeFields()
+    {
+        foreach ($this->_segmentInfos as $segName => $segmentInfo) {
+            foreach ($segmentInfo->getFieldInfos() as $fieldInfo) {
+                $this->_fieldsMap[$segName][$fieldInfo->number] = $this->_writer->addFieldInfo($fieldInfo);
+            }
+        }
+    }
+
+    /**
+     * Merge field's normalization factors
+     */
+    private function _mergeNorms()
+    {
+        foreach ($this->_writer->getFieldInfos() as $fieldInfo) {
+            if ($fieldInfo->isIndexed) {
+                foreach ($this->_segmentInfos as $segName => $segmentInfo) {
+                    if ($segmentInfo->hasDeletions()) {
+                        $srcNorm = $segmentInfo->normVector($fieldInfo->name);
+                        $norm    = '';
+                        $docs    = $segmentInfo->count();
+                        for ($count = 0; $count < $docs; $count++) {
+                            if (!$segmentInfo->isDeleted($count)) {
+                                $norm .= $srcNorm[$count];
+                            }
+                        }
+                        $this->_writer->addNorm($fieldInfo->name, $norm);
+                    } else {
+                        $this->_writer->addNorm($fieldInfo->name, $segmentInfo->normVector($fieldInfo->name));
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Merge fields information
+     */
+    private function _mergeStoredFields()
+    {
+        $this->_docCount = 0;
+
+        foreach ($this->_segmentInfos as $segName => $segmentInfo) {
+            $fdtFile = $segmentInfo->openCompoundFile('.fdt');
+
+            for ($count = 0; $count < $segmentInfo->count(); $count++) {
+                $fieldCount = $fdtFile->readVInt();
+                $storedFields = array();
+
+                for ($count2 = 0; $count2 < $fieldCount; $count2++) {
+                    $fieldNum = $fdtFile->readVInt();
+                    $bits = $fdtFile->readByte();
+                    $fieldInfo = $segmentInfo->getField($fieldNum);
+
+                    if (!($bits & 2)) { // Text data
+                        $storedFields[] =
+                                 new Zend_Search_Lucene_Field($fieldInfo->name,
+                                                              $fdtFile->readString(),
+                                                              'UTF-8',
+                                                              true,
+                                                              $fieldInfo->isIndexed,
+                                                              $bits & 1 );
+                    } else {            // Binary data
+                        $storedFields[] =
+                                 new Zend_Search_Lucene_Field($fieldInfo->name,
+                                                              $fdtFile->readBinary(),
+                                                              '',
+                                                              true,
+                                                              $fieldInfo->isIndexed,
+                                                              $bits & 1,
+                                                              true);
+                    }
+                }
+
+                if (!$segmentInfo->isDeleted($count)) {
+                    $this->_docCount++;
+                    $this->_writer->addStoredFields($storedFields);
+                }
+            }
+        }
+    }
+
+
+    /**
+     * Merge fields information
+     */
+    private function _mergeTerms()
+    {
+        $segmentInfoQueue = new Zend_Search_Lucene_Index_SegmentInfoPriorityQueue();
+
+        $segmentStartId = 0;
+        foreach ($this->_segmentInfos as $segName => $segmentInfo) {
+            $segmentStartId = $segmentInfo->reset($segmentStartId, true);
+
+            // Skip "empty" segments
+            if ($segmentInfo->currentTerm() !== null) {
+                $segmentInfoQueue->put($segmentInfo);
+            }
+        }
+
+        $this->_writer->initializeDictionaryFiles();
+
+        $termDocs = array();
+        while (($segmentInfo = $segmentInfoQueue->pop()) !== null) {
+            // Merge positions array
+            $termDocs += $segmentInfo->currentTermPositions();
+
+            if ($segmentInfoQueue->top() === null ||
+                $segmentInfoQueue->top()->currentTerm()->key() !=
+                            $segmentInfo->currentTerm()->key()) {
+                // We got new term
+                ksort($termDocs, SORT_NUMERIC);
+
+                // Add term if it's contained in any document
+                if (count($termDocs) > 0) {
+                    $this->_writer->addTerm($segmentInfo->currentTerm(), $termDocs);
+                }
+                $termDocs = array();
+            }
+
+            $segmentInfo->nextTerm();
+            // check, if segment dictionary is finished
+            if ($segmentInfo->currentTerm() !== null) {
+                // Put segment back into the priority queue
+                $segmentInfoQueue->put($segmentInfo);
+            }
+        }
+
+        $this->_writer->closeDictionaryFiles();
+    }
+}

search/Zend/Search/Lucene/Index/SegmentWriter.php

@@ -15,29 +15,26 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Index
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Exception */
-require_once 'Zend/Search/Lucene/Exception.php';
-
-/** Zend_Search_Lucene_Analysis_Analyzer */
-require_once 'Zend/Search/Lucene/Analysis/Analyzer.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
 
 /** Zend_Search_Lucene_Index_SegmentInfo */
-require_once 'Zend/Search/Lucene/Index/SegmentInfo.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Index
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
-class Zend_Search_Lucene_Index_SegmentWriter
+abstract class Zend_Search_Lucene_Index_SegmentWriter
 {
     /**
      * Expert: The fraction of terms in the "dictionary" which should be stored
@@ -48,7 +45,7 @@ class Zend_Search_Lucene_Index_SegmentWriter
      *
      * @var integer
      */
-    static public $indexInterval = 128;
+    public static $indexInterval = 128;
 
     /** Expert: The fraction of TermDocs entries stored in skip tables.
      * Larger values result in smaller indexes, greater acceleration, but fewer
@@ -61,28 +58,28 @@ class Zend_Search_Lucene_Index_SegmentWriter
      *
      * @var integer
      */
-    static public $skipInterval = 0x7FFFFFFF;
+    public static $skipInterval = 0x7FFFFFFF;
 
     /**
      * Number of docs in a segment
      *
      * @var integer
      */
-    private $_docCount;
+    protected $_docCount = 0;
 
     /**
      * Segment name
      *
      * @var string
      */
-    private $_name;
+    protected $_name;
 
     /**
      * File system adapter.
      *
      * @var Zend_Search_Lucene_Storage_Directory
      */
-    private $_directory;
+    protected $_directory;
 
     /**
      * List of the index files.
@@ -90,52 +87,41 @@ class Zend_Search_Lucene_Index_SegmentWriter
      *
      * @var unknown_type
      */
-    private $_files;
-
-    /**
-     * Term Dictionary
-     * Array of the Zend_Search_Lucene_Index_Term objects
-     * Corresponding Zend_Search_Lucene_Index_TermInfo object stored in the $_termDictionaryInfos
-     *
-     * @var array
-     */
-    private $_termDictionary;
-
-    /**
-     * Documents, which contain the term
-     *
-     * @var array
-     */
-    private $_termDocs;
+    protected $_files = array();
 
     /**
      * Segment fields. Array of Zend_Search_Lucene_Index_FieldInfo objects for this segment
      *
      * @var array
      */
-    private $_fields;
+    protected $_fields = array();
 
     /**
-     * Sizes of the indexed fields.
-     * Used for normalization factors calculation.
+     * Normalization factors.
+     * An array fieldName => normVector
+     * normVector is a binary string.
+     * Each byte corresponds to an indexed document in a segment and
+     * encodes normalization factor (float value, encoded by
+     * Zend_Search_Lucene_Search_Similarity::encodeNorm())
      *
      * @var array
      */
-    private $_fieldLengths;
+    protected $_norms = array();
+
 
     /**
      * '.fdx'  file - Stored Fields, the field index.
      *
      * @var Zend_Search_Lucene_Storage_File
      */
-    private $_fdxFile;
+    protected $_fdxFile = null;
 
     /**
      * '.fdt'  file - Stored Fields, the field data.
      *
      * @var Zend_Search_Lucene_Storage_File
      */
-    private $_fdtFile;
+    protected $_fdtFile = null;
 
 
     /**
@@ -144,132 +130,125 @@ class Zend_Search_Lucene_Index_SegmentWriter
      * @param Zend_Search_Lucene_Storage_Directory $directory
      * @param string $name
      */
-    public function __construct($directory, $name)
+    public function __construct(Zend_Search_Lucene_Storage_Directory $directory, $name)
     {
         $this->_directory = $directory;
         $this->_name      = $name;
-        $this->_docCount  = 0;
-
-        $this->_fields         = array();
-        $this->_termDocs       = array();
-        $this->_files          = array();
-        $this->_norms          = array();
-        $this->_fieldLengths   = array();
-        $this->_termDictionary = array();
-
-        $this->_fdxFile = null;
-        $this->_fdtFile = null;
     }
 
 
     /**
      * Add field to the segment
      *
+     * Returns actual field number
+     *
      * @param Zend_Search_Lucene_Field $field
+     * @return integer
      */
-    private function _addFieldInfo(Zend_Search_Lucene_Field $field)
+    public function addField(Zend_Search_Lucene_Field $field)
     {
         if (!isset($this->_fields[$field->name])) {
+            $fieldNumber = count($this->_fields);
             $this->_fields[$field->name] =
                                 new Zend_Search_Lucene_Index_FieldInfo($field->name,
                                                                        $field->isIndexed,
-                                                                       count($this->_fields),
+                                                                       $fieldNumber,
                                                                        $field->storeTermVector);
+
+            return $fieldNumber;
         } else {
             $this->_fields[$field->name]->isIndexed       |= $field->isIndexed;
             $this->_fields[$field->name]->storeTermVector |= $field->storeTermVector;
+
+            return $this->_fields[$field->name]->number;
         }
     }
 
+    /**
+     * Add fieldInfo to the segment
+     *
+     * Returns actual field number
+     *
+     * @param Zend_Search_Lucene_Index_FieldInfo $fieldInfo
+     * @return integer
+     */
+    public function addFieldInfo(Zend_Search_Lucene_Index_FieldInfo $fieldInfo)
+    {
+        if (!isset($this->_fields[$fieldInfo->name])) {
+            $fieldNumber = count($this->_fields);
+            $this->_fields[$fieldInfo->name] =
+                                new Zend_Search_Lucene_Index_FieldInfo($fieldInfo->name,
+                                                                       $fieldInfo->isIndexed,
+                                                                       $fieldNumber,
+                                                                       $fieldInfo->storeTermVector);
+
+            return $fieldNumber;
+        } else {
+            $this->_fields[$fieldInfo->name]->isIndexed       |= $fieldInfo->isIndexed;
+            $this->_fields[$fieldInfo->name]->storeTermVector |= $fieldInfo->storeTermVector;
+
+            return $this->_fields[$fieldInfo->name]->number;
+        }
+    }
 
     /**
-     * Adds a document to this segment.
+     * Returns array of FieldInfo objects.
      *
-     * @param Zend_Search_Lucene_Document $document
-     * @throws Zend_Search_Lucene_Exception
+     * @return array
      */
-    public function addDocument(Zend_Search_Lucene_Document $document)
+    public function getFieldInfos()
     {
-        $storedFields = array();
+        return $this->_fields;
+    }
 
-        foreach ($document->getFieldNames() as $fieldName) {
-            $field = $document->getField($fieldName);
-            $this->_addFieldInfo($field);
+    /**
+     * Add stored fields information
+     *
+     * @param array $storedFields array of Zend_Search_Lucene_Field objects
+     */
+    public function addStoredFields($storedFields)
+    {
+        if (!isset($this->_fdxFile)) {
+            $this->_fdxFile = $this->_directory->createFile($this->_name . '.fdx');
+            $this->_fdtFile = $this->_directory->createFile($this->_name . '.fdt');
 
-            if ($field->storeTermVector) {
-                /**
-                 * @todo term vector storing support
-                 */
-                throw new Zend_Search_Lucene_Exception('Store term vector functionality is not supported yet.');
-            }
-
-            if ($field->isIndexed) {
-                if ($field->isTokenized) {
-                    $tokenList = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($field->stringValue);
-                } else {
-                    $tokenList = array();
-                    $tokenList[] = new Zend_Search_Lucene_Analysis_Token($field->stringValue, 0, strlen($field->stringValue));
-                }
-                $this->_fieldLengths[$field->name][$this->_docCount] = count($tokenList);
-
-                $position = 0;
-                foreach ($tokenList as $token) {
-                    $term = new Zend_Search_Lucene_Index_Term($token->getTermText(), $field->name);
-                    $termKey = $term->key();
-
-                    if (!isset($this->_termDictionary[$termKey])) {
-                        // New term
-                        $this->_termDictionary[$termKey] = $term;
-                        $this->_termDocs[$termKey] = array();
-                        $this->_termDocs[$termKey][$this->_docCount] = array();
-                    } else if (!isset($this->_termDocs[$termKey][$this->_docCount])) {
-                        // Existing term, but new term entry
-                        $this->_termDocs[$termKey][$this->_docCount] = array();
-                    }
-                    $position += $token->getPositionIncrement();
-                    $this->_termDocs[$termKey][$this->_docCount][] = $position;
-                }
-            }
-
-            if ($field->isStored) {
-                $storedFields[] = $field;
-            }
+            $this->_files[] = $this->_name . '.fdx';
+            $this->_files[] = $this->_name . '.fdt';
         }
 
-        if (count($storedFields) != 0) {
-            if (!isset($this->_fdxFile)) {
-                $this->_fdxFile = $this->_directory->createFile($this->_name . '.fdx');
-                $this->_fdtFile = $this->_directory->createFile($this->_name . '.fdt');
-
-                $this->_files[] = $this->_name . '.fdx';
-                $this->_files[] = $this->_name . '.fdt';
-            }
-
-            $this->_fdxFile->writeLong($this->_fdtFile->tell());
-            $this->_fdtFile->writeVInt(count($storedFields));
-            foreach ($storedFields as $field) {
-                $this->_fdtFile->writeVInt($this->_fields[$field->name]->number);
-                $fieldBits = ($field->isTokenized ? 0x01 : 0x00) |
-                             ($field->isBinary ?    0x02 : 0x00) |
-                             0x00; /* 0x04 - third bit, compressed (ZLIB) */
-                $this->_fdtFile->writeByte($fieldBits);
-                if ($field->isBinary) {
-                    $this->_fdtFile->writeVInt(strlen($field->stringValue));
-                    $this->_fdtFile->writeBytes($field->stringValue);
-                } else {
-                    $this->_fdtFile->writeString($field->stringValue);
-                }
+        $this->_fdxFile->writeLong($this->_fdtFile->tell());
+        $this->_fdtFile->writeVInt(count($storedFields));
+        foreach ($storedFields as $field) {
+            $this->_fdtFile->writeVInt($this->_fields[$field->name]->number);
+            $fieldBits = ($field->isTokenized ? 0x01 : 0x00) |
+                         ($field->isBinary ?    0x02 : 0x00) |
+                         0x00; /* 0x04 - third bit, compressed (ZLIB) */
+            $this->_fdtFile->writeByte($fieldBits);
+            if ($field->isBinary) {
+                $this->_fdtFile->writeVInt(strlen($field->value));
+                $this->_fdtFile->writeBytes($field->value);
+            } else {
+                $this->_fdtFile->writeString($field->getUtf8Value());
             }
         }
 
         $this->_docCount++;
     }
 
+    /**
+     * Returns the total number of documents in this segment.
+     *
+     * @return integer
+     */
+    public function count()
+    {
+        return $this->_docCount;
+    }
 
     /**
      * Dump Field Info (.fnm) segment file
      */
-    private function _dumpFNM()
+    protected function _dumpFNM()
     {
         $fnmFile = $this->_directory->createFile($this->_name . '.fnm');
         $fnmFile->writeVInt(count($this->_fields));
@@ -283,20 +262,9 @@ class Zend_Search_Lucene_Index_SegmentWriter
                                );
 
             if ($field->isIndexed) {
-                $fieldNum   = $this->_fields[$field->name]->number;
-                $fieldName  = $field->name;
-                $similarity = Zend_Search_Lucene_Search_Similarity::getDefault();
-                $norm       = '';
-
-                for ($count = 0; $count < $this->_docCount; $count++) {
-                    $numTokens = isset($this->_fieldLengths[$fieldName][$count]) ?
-                                      $this->_fieldLengths[$fieldName][$count] : 0;
-                    $norm .= chr($similarity->encodeNorm($similarity->lengthNorm($fieldName, $numTokens)));
-                }
-
-                $normFileName = $this->_name . '.f' . $fieldNum;
+                $normFileName = $this->_name . '.f' . $field->number;
                 $fFile = $this->_directory->createFile($normFileName);
-                $fFile->writeBytes($norm);
+                $fFile->writeBytes($this->_norms[$field->name]);
                 $this->_files[] = $normFileName;
             }
         }
@@ -305,6 +273,194 @@ class Zend_Search_Lucene_Index_SegmentWriter
     }
 
 
+
+    /**
+     * Term Dictionary file
+     *
+     * @var Zend_Search_Lucene_Storage_File
+     */
+    private $_tisFile = null;
+
+    /**
+     * Term Dictionary index file
+     *
+     * @var Zend_Search_Lucene_Storage_File
+     */
+    private $_tiiFile = null;
+
+    /**
+     * Frequencies file
+     *
+     * @var Zend_Search_Lucene_Storage_File
+     */
+    private $_frqFile = null;
+
+    /**
+     * Positions file
+     *
+     * @var Zend_Search_Lucene_Storage_File
+     */
+    private $_prxFile = null;
+
+    /**
+     * Number of written terms
+     *
+     * @var integer
+     */
+    private $_termCount;
+
+
+    /**
+     * Last saved term
+     *
+     * @var Zend_Search_Lucene_Index_Term
+     */
+    private $_prevTerm;
+
+    /**
+     * Last saved term info
+     *
+     * @var Zend_Search_Lucene_Index_TermInfo
+     */
+    private $_prevTermInfo;
+
+    /**
+     * Last saved index term
+     *
+     * @var Zend_Search_Lucene_Index_Term
+     */
+    private $_prevIndexTerm;
+
+    /**
+     * Last saved index term info
+     *
+     * @var Zend_Search_Lucene_Index_TermInfo
+     */
+    private $_prevIndexTermInfo;
+
+    /**
+     * Last term dictionary file position
+     *
+     * @var integer
+     */
+    private $_lastIndexPosition;
+
+    /**
+     * Create dicrionary, frequency and positions files and write necessary headers
+     */
+    public function initializeDictionaryFiles()
+    {
+        $this->_tisFile = $this->_directory->createFile($this->_name . '.tis');
+        $this->_tisFile->writeInt((int)0xFFFFFFFE);
+        $this->_tisFile->writeLong(0 /* dummy data for terms count */);
+        $this->_tisFile->writeInt(self::$indexInterval);
+        $this->_tisFile->writeInt(self::$skipInterval);
+
+        $this->_tiiFile = $this->_directory->createFile($this->_name . '.tii');
+        $this->_tiiFile->writeInt((int)0xFFFFFFFE);
+        $this->_tiiFile->writeLong(0 /* dummy data for terms count */);
+        $this->_tiiFile->writeInt(self::$indexInterval);
+        $this->_tiiFile->writeInt(self::$skipInterval);
+
+        /** Dump dictionary header */
+        $this->_tiiFile->writeVInt(0);                    // preffix length
+        $this->_tiiFile->writeString('');                 // suffix
+        $this->_tiiFile->writeInt((int)0xFFFFFFFF);       // field number
+        $this->_tiiFile->writeByte((int)0x0F);
+        $this->_tiiFile->writeVInt(0);                    // DocFreq
+        $this->_tiiFile->writeVInt(0);                    // FreqDelta
+        $this->_tiiFile->writeVInt(0);                    // ProxDelta
+        $this->_tiiFile->writeVInt(20);                   // IndexDelta
+
+        $this->_frqFile = $this->_directory->createFile($this->_name . '.frq');
+        $this->_prxFile = $this->_directory->createFile($this->_name . '.prx');
+
+        $this->_files[] = $this->_name . '.tis';
+        $this->_files[] = $this->_name . '.tii';
+        $this->_files[] = $this->_name . '.frq';
+        $this->_files[] = $this->_name . '.prx';
+
+        $this->_prevTerm          = null;
+        $this->_prevTermInfo      = null;
+        $this->_prevIndexTerm     = null;
+        $this->_prevIndexTermInfo = null;
+        $this->_lastIndexPosition = 20;
+        $this->_termCount         = 0;
+
+    }
+
+    /**
+     * Add term
+     *
+     * Term positions is an array( docId => array(pos1, pos2, pos3, ...), ... )
+     *
+     * @param Zend_Search_Lucene_Index_Term $termEntry
+     * @param array $termDocs
+     */
+    public function addTerm($termEntry, $termDocs)
+    {
+        $freqPointer = $this->_frqFile->tell();
+        $proxPointer = $this->_prxFile->tell();
+
+        $prevDoc = 0;
+        foreach ($termDocs as $docId => $termPositions) {
+            $docDelta = ($docId - $prevDoc)*2;
+            $prevDoc = $docId;
+            if (count($termPositions) > 1) {
+                $this->_frqFile->writeVInt($docDelta);
+                $this->_frqFile->writeVInt(count($termPositions));
+            } else {
+                $this->_frqFile->writeVInt($docDelta + 1);
+            }
+
+            $prevPosition = 0;
+            foreach ($termPositions as $position) {
+                $this->_prxFile->writeVInt($position - $prevPosition);
+                $prevPosition = $position;
+            }
+        }
+
+        if (count($termDocs) >= self::$skipInterval) {
+            /**
+             * @todo Write Skip Data to a freq file.
+             * It's not used now, but make index more optimal
+             */
+            $skipOffset = $this->_frqFile->tell() - $freqPointer;
+        } else {
+            $skipOffset = 0;
+        }
+
+        $term = new Zend_Search_Lucene_Index_Term($termEntry->text,
+                                                  $this->_fields[$termEntry->field]->number);
+        $termInfo = new Zend_Search_Lucene_Index_TermInfo(count($termDocs),
+                                                          $freqPointer, $proxPointer, $skipOffset);
+
+        $this->_dumpTermDictEntry($this->_tisFile, $this->_prevTerm, $term, $this->_prevTermInfo, $termInfo);
+
+        if (($this->_termCount + 1) % self::$indexInterval == 0) {
+            $this->_dumpTermDictEntry($this->_tiiFile, $this->_prevIndexTerm, $term, $this->_prevIndexTermInfo, $termInfo);
+
+            $indexPosition = $this->_tisFile->tell();
+            $this->_tiiFile->writeVInt($indexPosition - $this->_lastIndexPosition);
+            $this->_lastIndexPosition = $indexPosition;
+
+        }
+        $this->_termCount++;
+    }
+
+    /**
+     * Close dictionary
+     */
+    public function closeDictionaryFiles()
+    {
+        $this->_tisFile->seek(4);
+        $this->_tisFile->writeLong($this->_termCount);
+
+        $this->_tiiFile->seek(4);
+        $this->_tiiFile->writeLong(ceil(($this->_termCount + 2)/self::$indexInterval));
+    }
+
+
     /**
      * Dump Term Dictionary segment file entry.
      * Used to write entry to .tis or .tii files
@@ -315,22 +471,47 @@ class Zend_Search_Lucene_Index_SegmentWriter
      * @param Zend_Search_Lucene_Index_TermInfo $prevTermInfo
      * @param Zend_Search_Lucene_Index_TermInfo $termInfo
      */
-    private function _dumpTermDictEntry(Zend_Search_Lucene_Storage_File $dicFile,
+    protected function _dumpTermDictEntry(Zend_Search_Lucene_Storage_File $dicFile,
                                         &$prevTerm,     Zend_Search_Lucene_Index_Term     $term,
                                         &$prevTermInfo, Zend_Search_Lucene_Index_TermInfo $termInfo)
     {
         if (isset($prevTerm) && $prevTerm->field == $term->field) {
-            $prefixLength = 0;
-            while ($prefixLength < strlen($prevTerm->text) &&
-                   $prefixLength < strlen($term->text) &&
-                   $prevTerm->text{$prefixLength} == $term->text{$prefixLength}
-                  ) {
-                $prefixLength++;
+            $matchedBytes = 0;
+            $maxBytes = min(strlen($prevTerm->text), strlen($term->text));
+            while ($matchedBytes < $maxBytes  &&
+                   $prevTerm->text[$matchedBytes] == $term->text[$matchedBytes]) {
+                $matchedBytes++;
             }
+
+            // Calculate actual matched UTF-8 pattern
+            $prefixBytes = 0;
+            $prefixChars = 0;
+            while ($prefixBytes < $matchedBytes) {
+                $charBytes = 1;
+                if ((ord($term->text[$prefixBytes]) & 0xC0) == 0xC0) {
+                    $charBytes++;
+                    if (ord($term->text[$prefixBytes]) & 0x20 ) {
+                        $charBytes++;
+                        if (ord($term->text[$prefixBytes]) & 0x10 ) {
+                            $charBytes++;
+                        }
+                    }
+                }
+
+                if ($prefixBytes + $charBytes > $matchedBytes) {
+                    // char crosses matched bytes boundary
+                    // skip char
+                    break;
+                }
+
+                $prefixChars++;
+                $prefixBytes += $charBytes;
+            }
+
             // Write preffix length
-            $dicFile->writeVInt($prefixLength);
+            $dicFile->writeVInt($prefixChars);
             // Write suffix
-            $dicFile->writeString( substr($term->text, $prefixLength) );
+            $dicFile->writeString(substr($term->text, $prefixBytes));
         } else {
             // Write preffix length
             $dicFile->writeVInt(0);
@@ -363,107 +544,11 @@ class Zend_Search_Lucene_Index_SegmentWriter
         $prevTermInfo = $termInfo;
     }
 
-    /**
-     * Dump Term Dictionary (.tis) and Term Dictionary Index (.tii) segment files
-     */
-    private function _dumpDictionary()
-    {
-        $termKeys = array_keys($this->_termDictionary);
-        sort($termKeys, SORT_STRING);
-
-        $tisFile = $this->_directory->createFile($this->_name . '.tis');
-        $tisFile->writeInt((int)0xFFFFFFFE);
-        $tisFile->writeLong(count($termKeys));
-        $tisFile->writeInt(self::$indexInterval);
-        $tisFile->writeInt(self::$skipInterval);
-
-        $tiiFile = $this->_directory->createFile($this->_name . '.tii');
-        $tiiFile->writeInt((int)0xFFFFFFFE);
-        $tiiFile->writeLong(ceil((count($termKeys) + 2)/self::$indexInterval));
-        $tiiFile->writeInt(self::$indexInterval);
-        $tiiFile->writeInt(self::$skipInterval);
-
-        /** Dump dictionary header */
-        $tiiFile->writeVInt(0);                    // preffix length
-        $tiiFile->writeString('');                 // suffix
-        $tiiFile->writeInt((int)0xFFFFFFFF);       // field number
-        $tiiFile->writeByte((int)0x0F);
-        $tiiFile->writeVInt(0);                    // DocFreq
-        $tiiFile->writeVInt(0);                    // FreqDelta
-        $tiiFile->writeVInt(0);                    // ProxDelta
-        $tiiFile->writeVInt(20);                   // IndexDelta
-
-        $frqFile = $this->_directory->createFile($this->_name . '.frq');
-        $prxFile = $this->_directory->createFile($this->_name . '.prx');
-
-        $termCount = 1;
-
-        $prevTerm     = null;
-        $prevTermInfo = null;
-        $prevIndexTerm     = null;
-        $prevIndexTermInfo = null;
-        $prevIndexPosition = 20;
-
-        foreach ($termKeys as $termId) {
-            $freqPointer = $frqFile->tell();
-            $proxPointer = $prxFile->tell();
-
-            $prevDoc = 0;
-            foreach ($this->_termDocs[$termId] as $docId => $termPositions) {
-                $docDelta = ($docId - $prevDoc)*2;
-                $prevDoc = $docId;
-                if (count($termPositions) > 1) {
-                    $frqFile->writeVInt($docDelta);
-                    $frqFile->writeVInt(count($termPositions));
-                } else {
-                    $frqFile->writeVInt($docDelta + 1);
-                }
-
-                $prevPosition = 0;
-                foreach ($termPositions as $position) {
-                    $prxFile->writeVInt($position - $prevPosition);
-                    $prevPosition = $position;
-                }
-            }
-
-            if (count($this->_termDocs[$termId]) >= self::$skipInterval) {
-                /**
-                 * @todo Write Skip Data to a freq file.
-                 * It's not used now, but make index more optimal
-                 */
-                $skipOffset = $frqFile->tell() - $freqPointer;
-            } else {
-                $skipOffset = 0;
-            }
-
-            $term = new Zend_Search_Lucene_Index_Term($this->_termDictionary[$termId]->text,
-                                                      $this->_fields[$this->_termDictionary[$termId]->field]->number);
-            $termInfo = new Zend_Search_Lucene_Index_TermInfo(count($this->_termDocs[$termId]),
-                                            $freqPointer, $proxPointer, $skipOffset);
-
-            $this->_dumpTermDictEntry($tisFile, $prevTerm, $term, $prevTermInfo, $termInfo);
-
-            if ($termCount % self::$indexInterval == 0) {
-                $this->_dumpTermDictEntry($tiiFile, $prevIndexTerm, $term, $prevIndexTermInfo, $termInfo);
-
-                $indexPosition = $tisFile->tell();
-                $tiiFile->writeVInt($indexPosition - $prevIndexPosition);
-                $prevIndexPosition = $indexPosition;
-            }
-            $termCount++;
-        }
-
-        $this->_files[] = $this->_name . '.tis';
-        $this->_files[] = $this->_name . '.tii';
-        $this->_files[] = $this->_name . '.frq';
-        $this->_files[] = $this->_name . '.prx';
-    }
-
 
     /**
      * Generate compound index file
      */
-    private function _generateCFS()
+    protected function _generateCFS()
     {
         $cfsFile = $this->_directory->createFile($this->_name . '.cfs');
         $cfsFile->writeVInt(count($this->_files));
@@ -486,8 +571,13 @@ class Zend_Search_Lucene_Index_SegmentWriter
             $cfsFile->seek($dataOffset);
 
             $dataFile = $this->_directory->getFileObject($fileName);
-            $data = $dataFile->readBytes($this->_directory->fileLength($fileName));
-            $cfsFile->writeBytes($data);
+
+            $byteCount = $this->_directory->fileLength($fileName);
+            while ($byteCount > 0) {
+                $data = $dataFile->readBytes(min($byteCount, 131072 /*128Kb*/));
+                $byteCount -= strlen($data);
+                $cfsFile->writeBytes($data);
+            }
 
             $this->_directory->deleteFile($fileName);
         }
@@ -499,21 +589,6 @@ class Zend_Search_Lucene_Index_SegmentWriter
      *
      * @return Zend_Search_Lucene_Index_SegmentInfo
      */
-    public function close()
-    {
-        if ($this->_docCount == 0) {
-            return null;
-        }
-
-        $this->_dumpFNM();
-        $this->_dumpDictionary();
-
-        $this->_generateCFS();
-
-        return new Zend_Search_Lucene_Index_SegmentInfo($this->_name,
-                                                        $this->_docCount,
-                                                        $this->_directory);
-    }
-
+    abstract public function close();
 }
 

search/Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php

@@ -0,0 +1,213 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Index
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Exception */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+/** Zend_Search_Lucene_Analysis_Analyzer */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php';
+
+/** Zend_Search_Lucene_Index_SegmentWriter */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentWriter.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Index
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter extends Zend_Search_Lucene_Index_SegmentWriter
+{
+    /**
+     * Term Dictionary
+     * Array of the Zend_Search_Lucene_Index_Term objects
+     * Corresponding Zend_Search_Lucene_Index_TermInfo object stored in the $_termDictionaryInfos
+     *
+     * @var array
+     */
+    protected $_termDictionary;
+
+    /**
+     * Documents, which contain the term
+     *
+     * @var array
+     */
+    protected $_termDocs;
+
+    /**
+     * Object constructor.
+     *
+     * @param Zend_Search_Lucene_Storage_Directory $directory
+     * @param string $name
+     */
+    public function __construct(Zend_Search_Lucene_Storage_Directory $directory, $name)
+    {
+        parent::__construct($directory, $name);
+
+        $this->_termDocs       = array();
+        $this->_termDictionary = array();
+    }
+
+
+    /**
+     * Adds a document to this segment.
+     *
+     * @param Zend_Search_Lucene_Document $document
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public function addDocument(Zend_Search_Lucene_Document $document)
+    {
+        $storedFields = array();
+        $docNorms     = array();
+        $similarity   = Zend_Search_Lucene_Search_Similarity::getDefault();
+
+        foreach ($document->getFieldNames() as $fieldName) {
+            $field = $document->getField($fieldName);
+            $this->addField($field);
+
+            if ($field->storeTermVector) {
+                /**
+                 * @todo term vector storing support
+                 */
+                throw new Zend_Search_Lucene_Exception('Store term vector functionality is not supported yet.');
+            }
+
+            if ($field->isIndexed) {
+                if ($field->isTokenized) {
+                    $analyzer = Zend_Search_Lucene_Analysis_Analyzer::getDefault();
+                    $analyzer->setInput($field->value, $field->encoding);
+
+                    $position     = 0;
+                    $tokenCounter = 0;
+                    while (($token = $analyzer->nextToken()) !== null) {
+                        $tokenCounter++;
+
+                        $term = new Zend_Search_Lucene_Index_Term($token->getTermText(), $field->name);
+                        $termKey = $term->key();
+
+                        if (!isset($this->_termDictionary[$termKey])) {
+                            // New term
+                            $this->_termDictionary[$termKey] = $term;
+                            $this->_termDocs[$termKey] = array();
+                            $this->_termDocs[$termKey][$this->_docCount] = array();
+                        } else if (!isset($this->_termDocs[$termKey][$this->_docCount])) {
+                            // Existing term, but new term entry
+                            $this->_termDocs[$termKey][$this->_docCount] = array();
+                        }
+                        $position += $token->getPositionIncrement();
+                        $this->_termDocs[$termKey][$this->_docCount][] = $position;
+                    }
+
+                    $docNorms[$field->name] = chr($similarity->encodeNorm( $similarity->lengthNorm($field->name,
+                                                                                                   $tokenCounter)*
+                                                                           $document->boost*
+                                                                           $field->boost ));
+                } else {
+                    $term = new Zend_Search_Lucene_Index_Term($field->getUtf8Value(), $field->name);
+                    $termKey = $term->key();
+
+                    if (!isset($this->_termDictionary[$termKey])) {
+                        // New term
+                        $this->_termDictionary[$termKey] = $term;
+                        $this->_termDocs[$termKey] = array();
+                        $this->_termDocs[$termKey][$this->_docCount] = array();
+                    } else if (!isset($this->_termDocs[$termKey][$this->_docCount])) {
+                        // Existing term, but new term entry
+                        $this->_termDocs[$termKey][$this->_docCount] = array();
+                    }
+                    $this->_termDocs[$termKey][$this->_docCount][] = 0; // position
+
+                    $docNorms[$field->name] = chr($similarity->encodeNorm( $similarity->lengthNorm($field->name, 1)*
+                                                                           $document->boost*
+                                                                           $field->boost ));
+                }
+            }
+
+            if ($field->isStored) {
+                $storedFields[] = $field;
+            }
+        }
+
+
+        foreach ($this->_fields as $fieldName => $field) {
+            if (!$field->isIndexed) {
+                continue;
+            }
+
+            if (!isset($this->_norms[$fieldName])) {
+                $this->_norms[$fieldName] = str_repeat(chr($similarity->encodeNorm( $similarity->lengthNorm($fieldName, 0) )),
+                                                       $this->_docCount);
+            }
+
+            if (isset($docNorms[$fieldName])){
+                $this->_norms[$fieldName] .= $docNorms[$fieldName];
+            } else {
+                $this->_norms[$fieldName] .= chr($similarity->encodeNorm( $similarity->lengthNorm($fieldName, 0) ));
+            }
+        }
+
+        $this->addStoredFields($storedFields);
+    }
+
+
+    /**
+     * Dump Term Dictionary (.tis) and Term Dictionary Index (.tii) segment files
+     */
+    protected function _dumpDictionary()
+    {
+        ksort($this->_termDictionary, SORT_STRING);
+
+        $this->initializeDictionaryFiles();
+
+        foreach ($this->_termDictionary as $termId => $term) {
+            $this->addTerm($term, $this->_termDocs[$termId]);
+        }
+
+        $this->closeDictionaryFiles();
+    }
+
+
+    /**
+     * Close segment, write it to disk and return segment info
+     *
+     * @return Zend_Search_Lucene_Index_SegmentInfo
+     */
+    public function close()
+    {
+        if ($this->_docCount == 0) {
+            return null;
+        }
+
+        $this->_dumpFNM();
+        $this->_dumpDictionary();
+
+        $this->_generateCFS();
+
+        return new Zend_Search_Lucene_Index_SegmentInfo($this->_name,
+                                                        $this->_docCount,
+                                                        $this->_directory);
+    }
+
+}
+

search/Zend/Search/Lucene/Index/SegmentWriter/StreamWriter.php

@@ -0,0 +1,94 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Index
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Exception */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+/** Zend_Search_Lucene_Index_SegmentInfo */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php';
+
+/** Zend_Search_Lucene_Index_SegmentWriter */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentWriter.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Index
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Index_SegmentWriter_StreamWriter extends Zend_Search_Lucene_Index_SegmentWriter
+{
+    /**
+     * Object constructor.
+     *
+     * @param Zend_Search_Lucene_Storage_Directory $directory
+     * @param string $name
+     */
+    public function __construct(Zend_Search_Lucene_Storage_Directory $directory, $name)
+    {
+        parent::__construct($directory, $name);
+    }
+
+
+    /**
+     * Create stored fields files and open them for write
+     */
+    public function createStoredFieldsFiles()
+    {
+        $this->_fdxFile = $this->_directory->createFile($this->_name . '.fdx');
+        $this->_fdtFile = $this->_directory->createFile($this->_name . '.fdt');
+
+        $this->_files[] = $this->_name . '.fdx';
+        $this->_files[] = $this->_name . '.fdt';
+    }
+
+    public function addNorm($fieldName, $normVector)
+    {
+        if (isset($this->_norms[$fieldName])) {
+            $this->_norms[$fieldName] .= $normVector;
+        } else {
+            $this->_norms[$fieldName] = $normVector;
+        }
+    }
+
+    /**
+     * Close segment, write it to disk and return segment info
+     *
+     * @return Zend_Search_Lucene_Index_SegmentInfo
+     */
+    public function close()
+    {
+        if ($this->_docCount == 0) {
+            return null;
+        }
+
+        $this->_dumpFNM();
+        $this->_generateCFS();
+
+        return new Zend_Search_Lucene_Index_SegmentInfo($this->_name,
+                                                        $this->_docCount,
+                                                        $this->_directory);
+    }
+}
+

search/Zend/Search/Lucene/Index/Term.php

@@ -15,7 +15,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Index
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
@@ -31,7 +31,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Index
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Index_Term
@@ -52,21 +52,57 @@ class Zend_Search_Lucene_Index_Term
 
 
     /**
-     * @todo docblock
+     * Object constructor
      */
-    public function __construct( $text, $field = 'contents' )
+    public function __construct($text, $field = null)
     {
-        $this->field = $field;
-        $this->text = $text;
+        $this->field = ($field === null)?  Zend_Search_Lucene::getDefaultSearchField() : $field;
+        $this->text  = $text;
     }
 
 
     /**
-     * @todo docblock
+     * Returns term key
+     *
+     * @return string
      */
     public function key()
     {
         return $this->field . chr(0) . $this->text;
     }
+
+    /**
+     * Get term prefix
+     *
+     * @param integer $length
+     * @return string
+     */
+    public static function getPrefix($str, $length)
+    {
+        $prefixBytes = 0;
+        $prefixChars = 0;
+        while ($prefixBytes < strlen($str)  &&  $prefixChars < $length) {
+            $charBytes = 1;
+            if ((ord($str[$prefixBytes]) & 0xC0) == 0xC0) {
+                $charBytes++;
+                if (ord($str[$prefixBytes]) & 0x20 ) {
+                    $charBytes++;
+                    if (ord($str[$prefixBytes]) & 0x10 ) {
+                        $charBytes++;
+                    }
+                }
+            }
+
+            if ($prefixBytes + $charBytes > strlen($str)) {
+                // wrong character
+                break;
+            }
+
+            $prefixChars++;
+            $prefixBytes += $charBytes;
+        }
+
+        return substr($str, 0, $prefixBytes);
+    }
 }
 

search/Zend/Search/Lucene/Index/TermInfo.php

@@ -15,7 +15,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Index
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
@@ -26,7 +26,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Index
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Index_TermInfo

search/Zend/Search/Lucene/Index/Writer.php

@@ -15,36 +15,80 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Index
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
-/** Zend_Search_Lucene_Index_SegmentWriter */
-require_once 'Zend/Search/Lucene/Index/SegmentWriter.php';
+/** Zend_Search_Lucene_Index_SegmentWriter_ */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php';
 
 /** Zend_Search_Lucene_Index_SegmentInfo */
-require_once 'Zend/Search/Lucene/Index/SegmentInfo.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php';
+
+/** Zend_Search_Lucene_Index_SegmentMerger */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentMerger.php';
+
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Index
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Index_Writer
 {
     /**
-     * @todo Implement segment merger
-     * @todo Implement mergeFactor, minMergeDocs, maxMergeDocs usage.
      * @todo Implement Analyzer substitution
      * @todo Implement Zend_Search_Lucene_Storage_DirectoryRAM and Zend_Search_Lucene_Storage_FileRAM to use it for
      *       temporary index files
      * @todo Directory lock processing
      */
 
+    /**
+     * Number of documents required before the buffered in-memory
+     * documents are written into a new Segment
+     *
+     * Default value is 10
+     *
+     * @var integer
+     */
+    public $maxBufferedDocs = 10;
+
+    /**
+     * Largest number of documents ever merged by addDocument().
+     * Small values (e.g., less than 10,000) are best for interactive indexing,
+     * as this limits the length of pauses while indexing to a few seconds.
+     * Larger values are best for batched indexing and speedier searches.
+     *
+     * Default value is PHP_INT_MAX
+     *
+     * @var integer
+     */
+    public $maxMergeDocs = PHP_INT_MAX;
+
+    /**
+     * Determines how often segment indices are merged by addDocument().
+     *
+     * With smaller values, less RAM is used while indexing,
+     * and searches on unoptimized indices are faster,
+     * but indexing speed is slower.
+     *
+     * With larger values, more RAM is used during indexing,
+     * and while searches on unoptimized indices are slower,
+     * indexing is faster.
+     *
+     * Thus larger values (> 10) are best for batch index creation,
+     * and smaller values (< 10) for indices that are interactively maintained.
+     *
+     * Default value is 10
+     *
+     * @var integer
+     */
+    public $mergeFactor = 10;
+
     /**
      * File system adapter.
      *
@@ -54,51 +98,11 @@ class Zend_Search_Lucene_Index_Writer
 
 
     /**
-     * Index version
-     * Counts how often the index has been changed by adding or deleting docs
+     * Changes counter.
      *
      * @var integer
      */
-    private $_version;
-
-    /**
-     * Segment name counter.
-     * Used to name new segments .
-     *
-     * @var integer
-     */
-    private $_segmentNameCounter;
-
-    /**
-     * Number of the segments in the index
-     *
-     * @var inteher
-     */
-    private $_segments;
-
-    /**
-     * Determines how often segment indices
-     * are merged by addDocument().
-     *
-     * @var integer
-     */
-    public $mergeFactor;
-
-    /**
-     * Determines the minimal number of documents required before
-     * the buffered in-memory documents are merging and a new Segment
-     * is created.
-     *
-     * @var integer
-     */
-    public $minMergeDocs;
-
-    /**
-     * Determines the largest number of documents ever merged by addDocument().
-     *
-     * @var integer
-     */
-    public $maxMergeDocs;
+    private $_versionUpdate = 0;
 
     /**
      * List of the segments, created by index writer
@@ -106,14 +110,30 @@ class Zend_Search_Lucene_Index_Writer
      *
      * @var array
      */
-    private $_newSegments;
+    private $_newSegments = array();
+
+    /**
+     * List of segments to be deleted on commit
+     *
+     * @var array
+     */
+    private $_segmentsToDelete = array();
 
     /**
      * Current segment to add documents
      *
-     * @var Zend_Search_Lucene_Index_SegmentWriter
+     * @var Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter
      */
-    private $_currentSegment;
+    private $_currentSegment = null;
+
+    /**
+     * Array of Zend_Search_Lucene_Index_SegmentInfo objects for this index.
+     *
+     * It's a reference to the corresponding Zend_Search_Lucene::$_segmentInfos array
+     *
+     * @var array Zend_Search_Lucene_Index_SegmentInfo
+     */
+    private $_segmentInfos;
 
     /**
      * List of indexfiles extensions
@@ -131,7 +151,8 @@ class Zend_Search_Lucene_Index_Writer
                                              '.tvx' => '.tvx',
                                              '.tvd' => '.tvd',
                                              '.tvf' => '.tvf',
-                                             '.del' => '.del'  );
+                                             '.del' => '.del',
+                                             '.sti' => '.sti' );
 
     /**
      * Opens the index for writing
@@ -142,11 +163,13 @@ class Zend_Search_Lucene_Index_Writer
      * index or overwrite the existing one.
      *
      * @param Zend_Search_Lucene_Storage_Directory $directory
+     * @param array $segmentInfos
      * @param boolean $create
      */
-    public function __construct(Zend_Search_Lucene_Storage_Directory $directory, $create = false)
+    public function __construct(Zend_Search_Lucene_Storage_Directory $directory, &$segmentInfos, $create = false)
     {
-        $this->_directory = $directory;
+        $this->_directory    = $directory;
+        $this->_segmentInfos = &$segmentInfos;
 
         if ($create) {
             foreach ($this->_directory->fileList() as $file) {
@@ -159,8 +182,13 @@ class Zend_Search_Lucene_Index_Writer
             }
             $segmentsFile = $this->_directory->createFile('segments');
             $segmentsFile->writeInt((int)0xFFFFFFFF);
-            // write version
-            $segmentsFile->writeLong(0);
+
+            // write version (is initialized by current time
+            // $segmentsFile->writeLong((int)microtime(true));
+            $version = microtime(true);
+            $segmentsFile->writeInt((int)($version/((double)0xFFFFFFFF + 1)));
+            $segmentsFile->writeInt((int)($version & 0xFFFFFFFF));
+
             // write name counter
             $segmentsFile->writeInt(0);
             // write segment counter
@@ -169,27 +197,13 @@ class Zend_Search_Lucene_Index_Writer
             $deletableFile = $this->_directory->createFile('deletable');
             // write counter
             $deletableFile->writeInt(0);
-
-            $this->_version            = 0;
-            $this->_segmentNameCounter = 0;
-            $this->_segments           = 0;
         } else {
             $segmentsFile = $this->_directory->getFileObject('segments');
             $format = $segmentsFile->readInt();
             if ($format != (int)0xFFFFFFFF) {
                 throw new Zend_Search_Lucene_Exception('Wrong segments file format');
             }
-
-            // read version
-            $this->_version            = $segmentsFile->readLong();
-            // read counter
-            $this->_segmentNameCounter = $segmentsFile->readInt();
-            // read segment counter
-            $this->_segments           = $segmentsFile->readInt();
         }
-
-        $this->_newSegments = array();
-        $this->_currentSegment = null;
     }
 
     /**
@@ -201,49 +215,218 @@ class Zend_Search_Lucene_Index_Writer
     {
         if ($this->_currentSegment === null) {
             $this->_currentSegment =
-                new Zend_Search_Lucene_Index_SegmentWriter($this->_directory, $this->_newSegmentName());
+                new Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter($this->_directory, $this->_newSegmentName());
         }
         $this->_currentSegment->addDocument($document);
-        $this->_version++;
+
+        if ($this->_currentSegment->count() >= $this->maxBufferedDocs) {
+            $this->commit();
+        }
+
+        $this->_versionUpdate++;
+
+        $this->_maybeMergeSegments();
     }
 
 
+    /**
+     * Merge segments if necessary
+     */
+    private function _maybeMergeSegments()
+    {
+        $segmentSizes = array();
+        foreach ($this->_segmentInfos as $segId => $segmentInfo) {
+            $segmentSizes[$segId] = $segmentInfo->count();
+        }
+
+        $mergePool   = array();
+        $poolSize    = 0;
+        $sizeToMerge = $this->maxBufferedDocs;
+        asort($segmentSizes, SORT_NUMERIC);
+        foreach ($segmentSizes as $segId => $size) {
+            // Check, if segment comes into a new merging block
+            while ($size >= $sizeToMerge) {
+                // Merge previous block if it's large enough
+                if ($poolSize >= $sizeToMerge) {
+                    $this->_mergeSegments($mergePool);
+                }
+                $mergePool   = array();
+                $poolSize    = 0;
+
+                $sizeToMerge *= $this->mergeFactor;
+
+                if ($sizeToMerge > $this->maxMergeDocs) {
+                    return;
+                }
+            }
+
+            $mergePool[] = $this->_segmentInfos[$segId];
+            $poolSize += $size;
+        }
+
+        if ($poolSize >= $sizeToMerge) {
+            $this->_mergeSegments($mergePool);
+        }
+    }
+
+    /**
+     * Merge specified segments
+     *
+     * $segments is an array of SegmentInfo objects
+     *
+     * @param array $segments
+     */
+    private function _mergeSegments($segments)
+    {
+        // Try to get exclusive non-blocking lock to the 'index.optimization.lock'
+        // Skip optimization if it's performed by other process right now
+        $optimizationLock = $this->_directory->createFile('index.optimization.lock');
+        if (!$optimizationLock->lock(LOCK_EX,true)) {
+            return;
+        }
+
+        $newName = $this->_newSegmentName();
+        $merger = new Zend_Search_Lucene_Index_SegmentMerger($this->_directory,
+                                                             $newName);
+        foreach ($segments as $segmentInfo) {
+            $merger->addSource($segmentInfo);
+            $this->_segmentsToDelete[$segmentInfo->getName()] = $segmentInfo->getName();
+        }
+
+        $newSegment = $merger->merge();
+        if ($newSegment !== null) {
+            $this->_newSegments[$newSegment->getName()] = $newSegment;
+        }
+
+        $this->commit();
+
+        // optimization is finished
+        $optimizationLock->unlock();
+    }
 
     /**
      * Update segments file by adding current segment to a list
-     * @todo !!!!!Finish the implementation
      *
      * @throws Zend_Search_Lucene_Exception
      */
     private function _updateSegments()
     {
-        $segmentsFile   = $this->_directory->getFileObject('segments');
-        $newSegmentFile = $this->_directory->createFile('segments.new');
-
-        $newSegmentFile->writeInt((int)0xFFFFFFFF);
-        $newSegmentFile->writeLong($this->_version);
-        $newSegmentFile->writeInt($this->_segmentNameCounter);
-
-        $this->_segments += count($this->_newSegments);
-        $newSegmentFile->writeInt($this->_segments);
-
-        $segmentsFile->seek(20);
-        $newSegmentFile->writeBytes($segmentsFile->readBytes($this->_directory->fileLength('segments') - 20));
-
-        foreach ($this->_newSegments as $segmentName => $segmentInfo) {
-            $newSegmentFile->writeString($segmentName);
-            $newSegmentFile->writeInt($segmentInfo->count());
+        // Get an exclusive index lock
+        // Wait, until all parallel searchers or indexers won't stop
+        // and stop all next searchers, while we are updating segments file
+        $lock = $this->_directory->getFileObject('index.lock');
+        if (!$lock->lock(LOCK_EX)) {
+            throw new Zend_Search_Lucene_Exception('Can\'t obtain exclusive index lock');
         }
 
+
+        // Do not share file handlers to get file updates from other sessions.
+        $segmentsFile   = $this->_directory->getFileObject('segments', false);
+        $newSegmentFile = $this->_directory->createFile('segments.new', false);
+
+        // Write format marker
+        $newSegmentFile->writeInt((int)0xFFFFFFFF);
+
+        // Write index version
+        $segmentsFile->seek(4, SEEK_CUR);
+        // $version = $segmentsFile->readLong() + $this->_versionUpdate;
+        // Process version on 32-bit platforms
+        $versionHigh = $segmentsFile->readInt();
+        $versionLow  = $segmentsFile->readInt();
+        $version = $versionHigh * ((double)0xFFFFFFFF + 1) +
+                   (($versionLow < 0)? (double)0xFFFFFFFF - (-1 - $versionLow) : $versionLow);
+        $version += $this->_versionUpdate;
+        $this->_versionUpdate = 0;
+        $newSegmentFile->writeInt((int)($version/((double)0xFFFFFFFF + 1)));
+        $newSegmentFile->writeInt((int)($version & 0xFFFFFFFF));
+
+        // Write segment name counter
+        $newSegmentFile->writeInt($segmentsFile->readInt());
+
+        // Get number of segments offset
+        $numOfSegmentsOffset = $newSegmentFile->tell();
+        // Write number of segemnts
+        $segmentsCount = $segmentsFile->readInt();
+        $newSegmentFile->writeInt(0);  // Write dummy data (segment counter)
+
+        $segments = array();
+        for ($count = 0; $count < $segmentsCount; $count++) {
+            $segName = $segmentsFile->readString();
+            $segSize = $segmentsFile->readInt();
+
+            if (!in_array($segName, $this->_segmentsToDelete)) {
+                $newSegmentFile->writeString($segName);
+                $newSegmentFile->writeInt($segSize);
+
+                $segments[$segName] = $segSize;
+            }
+        }
+        $segmentsFile->close();
+
+        $segmentsCount = count($segments) + count($this->_newSegments);
+
+        // Remove segments, not listed in $segments (deleted)
+        // Load segments, not listed in $this->_segmentInfos
+        foreach ($this->_segmentInfos as $segId => $segInfo) {
+            if (isset($segments[$segInfo->getName()])) {
+                // Segment is already included into $this->_segmentInfos
+                unset($segments[$segInfo->getName()]);
+            } else {
+                // remove deleted segment from a list
+                unset($this->_segmentInfos[$segId]);
+            }
+        }
+        // $segments contains a list of segments to load
+        // do it later
+
+        foreach ($this->_newSegments as $segName => $segmentInfo) {
+            $newSegmentFile->writeString($segName);
+            $newSegmentFile->writeInt($segmentInfo->count());
+
+            $this->_segmentInfos[] = $segmentInfo;
+        }
+        $this->_newSegments = array();
+
+        $newSegmentFile->seek($numOfSegmentsOffset);
+        $newSegmentFile->writeInt($segmentsCount);  // Update segments count
+        $newSegmentFile->close();
         $this->_directory->renameFile('segments.new', 'segments');
+
+
+        // Segments file update is finished
+        // Switch back to shared lock mode
+        $lock->lock(LOCK_SH);
+
+
+        $fileList = $this->_directory->fileList();
+        foreach ($this->_segmentsToDelete as $nameToDelete) {
+            foreach (self::$_indexExtensions as $ext) {
+                if ($this->_directory->fileExists($nameToDelete . $ext)) {
+                    $this->_directory->deleteFile($nameToDelete . $ext);
+                }
+            }
+
+            foreach ($fileList as $file) {
+                if (substr($file, 0, strlen($nameToDelete) + 2) == ($nameToDelete . '.f') &&
+                    ctype_digit( substr($file, strlen($nameToDelete) + 2) )) {
+                        $this->_directory->deleteFile($file);
+                    }
+            }
+        }
+        $this->_segmentsToDelete = array();
+
+        // Load segments, created by other process
+        foreach ($segments as $segName => $segSize) {
+            // Load new segments
+            $this->_segmentInfos[] = new Zend_Search_Lucene_Index_SegmentInfo($segName,
+                                                                              $segSize,
+                                                                              $this->_directory);
+        }
     }
 
 
     /**
      * Commit current changes
-     * returns array of new segments
-     *
-     * @return array
      */
     public function commit()
     {
@@ -255,14 +438,10 @@ class Zend_Search_Lucene_Index_Writer
             $this->_currentSegment = null;
         }
 
-        if (count($this->_newSegments) != 0) {
+        if (count($this->_newSegments)      != 0 ||
+            count($this->_segmentsToDelete) != 0) {
             $this->_updateSegments();
         }
-
-        $result = $this->_newSegments;
-        $this->_newSegments = array();
-
-        return $result;
     }
 
 
@@ -279,43 +458,16 @@ class Zend_Search_Lucene_Index_Writer
          */
     }
 
-
-    /**
-     * Returns the number of documents currently in this index.
-     *
-     * @return integer
-     */
-    public function docCount($readers)
-    {
-        /**
-         * @todo implementation
-         */
-    }
-
-
-    /**
-     * Flushes all changes to an index and closes all associated files.
-     *
-     */
-    public function close()
-    {
-        /**
-         * @todo implementation
-         */
-    }
-
-
     /**
      * Merges all segments together into a single segment, optimizing
      * an index for search.
+     * Input is an array of Zend_Search_Lucene_Index_SegmentInfo objects
      *
-     * return void
+     * @throws Zend_Search_Lucene_Exception
      */
     public function optimize()
     {
-        /**
-         * @todo implementation
-         */
+        $this->_mergeSegments($this->_segmentInfos);
     }
 
     /**
@@ -325,7 +477,30 @@ class Zend_Search_Lucene_Index_Writer
      */
     private function _newSegmentName()
     {
-        return '_' . base_convert($this->_segmentNameCounter++, 10, 36);
+        // Do not share file handler to get file updates from other sessions.
+        $segmentsFile = $this->_directory->getFileObject('segments', false);
+
+        // Get exclusive segments file lock
+        // We have guarantee, that we will not intersect with _updateSegments() call
+        // of other process, because it needs exclusive index lock and waits
+        // until all other searchers won't stop
+        if (!$segmentsFile->lock(LOCK_EX)) {
+            throw new Zend_Search_Lucene_Exception('Can\'t obtain exclusive index lock');
+        }
+
+        $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version)
+        $segmentNameCounter = $segmentsFile->readInt();
+
+        $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version)
+        $segmentsFile->writeInt($segmentNameCounter + 1);
+
+        // Flash output to guarantee that wrong value will not be loaded between unlock and
+        // return (which calls $segmentsFile destructor)
+        $segmentsFile->flush();
+
+        $segmentsFile->unlock();
+
+        return '_' . base_convert($segmentNameCounter, 10, 36);
     }
 
 }

search/Zend/Search/Lucene/Interface.php

@@ -0,0 +1,330 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+interface Zend_Search_Lucene_Interface
+{
+    /**
+     * Returns the Zend_Search_Lucene_Storage_Directory instance for this index.
+     *
+     * @return Zend_Search_Lucene_Storage_Directory
+     */
+    public function getDirectory();
+
+    /**
+     * Returns the total number of documents in this index (including deleted documents).
+     *
+     * @return integer
+     */
+    public function count();
+
+    /**
+     * Returns one greater than the largest possible document number.
+     * This may be used to, e.g., determine how big to allocate a structure which will have
+     * an element for every document number in an index.
+     *
+     * @return integer
+     */
+    public function maxDoc();
+
+    /**
+     * Returns the total number of non-deleted documents in this index.
+     *
+     * @return integer
+     */
+    public function numDocs();
+
+    /**
+     * Checks, that document is deleted
+     *
+     * @param integer $id
+     * @return boolean
+     * @throws Zend_Search_Lucene_Exception    Exception is thrown if $id is out of the range
+     */
+    public function isDeleted($id);
+
+    /**
+     * Set default search field.
+     *
+     * Null means, that search is performed through all fields by default
+     *
+     * Default value is null
+     *
+     * @param string $fieldName
+     */
+    public static function setDefaultSearchField($fieldName);
+
+    /**
+     * Get default search field.
+     *
+     * Null means, that search is performed through all fields by default
+     *
+     * @return string
+     */
+    public static function getDefaultSearchField();
+
+    /**
+     * Retrieve index maxBufferedDocs option
+     *
+     * maxBufferedDocs is a minimal number of documents required before
+     * the buffered in-memory documents are written into a new Segment
+     *
+     * Default value is 10
+     *
+     * @return integer
+     */
+    public function getMaxBufferedDocs();
+
+    /**
+     * Set index maxBufferedDocs option
+     *
+     * maxBufferedDocs is a minimal number of documents required before
+     * the buffered in-memory documents are written into a new Segment
+     *
+     * Default value is 10
+     *
+     * @param integer $maxBufferedDocs
+     */
+    public function setMaxBufferedDocs($maxBufferedDocs);
+
+    /**
+     * Retrieve index maxMergeDocs option
+     *
+     * maxMergeDocs is a largest number of documents ever merged by addDocument().
+     * Small values (e.g., less than 10,000) are best for interactive indexing,
+     * as this limits the length of pauses while indexing to a few seconds.
+     * Larger values are best for batched indexing and speedier searches.
+     *
+     * Default value is PHP_INT_MAX
+     *
+     * @return integer
+     */
+    public function getMaxMergeDocs();
+
+    /**
+     * Set index maxMergeDocs option
+     *
+     * maxMergeDocs is a largest number of documents ever merged by addDocument().
+     * Small values (e.g., less than 10,000) are best for interactive indexing,
+     * as this limits the length of pauses while indexing to a few seconds.
+     * Larger values are best for batched indexing and speedier searches.
+     *
+     * Default value is PHP_INT_MAX
+     *
+     * @param integer $maxMergeDocs
+     */
+    public function setMaxMergeDocs($maxMergeDocs);
+
+    /**
+     * Retrieve index mergeFactor option
+     *
+     * mergeFactor determines how often segment indices are merged by addDocument().
+     * With smaller values, less RAM is used while indexing,
+     * and searches on unoptimized indices are faster,
+     * but indexing speed is slower.
+     * With larger values, more RAM is used during indexing,
+     * and while searches on unoptimized indices are slower,
+     * indexing is faster.
+     * Thus larger values (> 10) are best for batch index creation,
+     * and smaller values (< 10) for indices that are interactively maintained.
+     *
+     * Default value is 10
+     *
+     * @return integer
+     */
+    public function getMergeFactor();
+
+    /**
+     * Set index mergeFactor option
+     *
+     * mergeFactor determines how often segment indices are merged by addDocument().
+     * With smaller values, less RAM is used while indexing,
+     * and searches on unoptimized indices are faster,
+     * but indexing speed is slower.
+     * With larger values, more RAM is used during indexing,
+     * and while searches on unoptimized indices are slower,
+     * indexing is faster.
+     * Thus larger values (> 10) are best for batch index creation,
+     * and smaller values (< 10) for indices that are interactively maintained.
+     *
+     * Default value is 10
+     *
+     * @param integer $maxMergeDocs
+     */
+    public function setMergeFactor($mergeFactor);
+
+    /**
+     * Performs a query against the index and returns an array
+     * of Zend_Search_Lucene_Search_QueryHit objects.
+     * Input is a string or Zend_Search_Lucene_Search_Query.
+     *
+     * @param mixed $query
+     * @return array Zend_Search_Lucene_Search_QueryHit
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public function find($query);
+
+    /**
+     * Returns a list of all unique field names that exist in this index.
+     *
+     * @param boolean $indexed
+     * @return array
+     */
+    public function getFieldNames($indexed = false);
+
+    /**
+     * Returns a Zend_Search_Lucene_Document object for the document
+     * number $id in this index.
+     *
+     * @param integer|Zend_Search_Lucene_Search_QueryHit $id
+     * @return Zend_Search_Lucene_Document
+     */
+    public function getDocument($id);
+
+    /**
+     * Returns true if index contain documents with specified term.
+     *
+     * Is used for query optimization.
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @return boolean
+     */
+    public function hasTerm(Zend_Search_Lucene_Index_Term $term);
+
+    /**
+     * Returns IDs of all the documents containing term.
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @return array
+     */
+    public function termDocs(Zend_Search_Lucene_Index_Term $term);
+
+    /**
+     * Returns an array of all term freqs.
+     * Return array structure: array( docId => freq, ...)
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @return integer
+     */
+    public function termFreqs(Zend_Search_Lucene_Index_Term $term);
+
+    /**
+     * Returns an array of all term positions in the documents.
+     * Return array structure: array( docId => array( pos1, pos2, ...), ...)
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @return array
+     */
+    public function termPositions(Zend_Search_Lucene_Index_Term $term);
+
+    /**
+     * Returns the number of documents in this index containing the $term.
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @return integer
+     */
+    public function docFreq(Zend_Search_Lucene_Index_Term $term);
+
+    /**
+     * Retrive similarity used by index reader
+     *
+     * @return Zend_Search_Lucene_Search_Similarity
+     */
+    public function getSimilarity();
+
+    /**
+     * Returns a normalization factor for "field, document" pair.
+     *
+     * @param integer $id
+     * @param string $fieldName
+     * @return float
+     */
+    public function norm($id, $fieldName);
+
+    /**
+     * Returns true if any documents have been deleted from this index.
+     *
+     * @return boolean
+     */
+    public function hasDeletions();
+
+    /**
+     * Deletes a document from the index.
+     * $id is an internal document id
+     *
+     * @param integer|Zend_Search_Lucene_Search_QueryHit $id
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public function delete($id);
+
+    /**
+     * Adds a document to this index.
+     *
+     * @param Zend_Search_Lucene_Document $document
+     */
+    public function addDocument(Zend_Search_Lucene_Document $document);
+
+    /**
+     * Commit changes resulting from delete() or undeleteAll() operations.
+     */
+    public function commit();
+
+    /**
+     * Optimize index.
+     *
+     * Merges all segments into one
+     */
+    public function optimize();
+
+    /**
+     * Returns an array of all terms in this index.
+     *
+     * @return array
+     */
+    public function terms();
+
+    /**
+     * Undeletes all documents currently marked as deleted in this index.
+     */
+    public function undeleteAll();
+
+
+    /**
+     * Add reference to the index object
+     *
+     * @internal
+     */
+    public function addReference();
+
+    /**
+     * Remove reference from the index object
+     *
+     * When reference count becomes zero, index is closed and resources are cleaned up
+     *
+     * @internal
+     */
+    public function removeReference();
+}

search/Zend/Search/Lucene/PriorityQueue.php

@@ -0,0 +1,170 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/**
+ * Abstract Priority Queue
+ *
+ * It implements a priority queue.
+ * Please go to "Data Structures and Algorithms",
+ * Aho, Hopcroft, and Ullman, Addison-Wesley, 1983 (corrected 1987 edition),
+ * for implementation details.
+ *
+ * It provides O(log(N)) time of put/pop operations, where N is a size of queue
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+abstract class Zend_Search_Lucene_PriorityQueue
+{
+    /**
+     * Queue heap
+     *
+     * Heap contains balanced partial ordered binary tree represented in array
+     * [0] - top of the tree
+     * [1] - first child of [0]
+     * [2] - second child of [0]
+     * ...
+     * [2*n + 1] - first child of [n]
+     * [2*n + 2] - second child of [n]
+     *
+     * @var array
+     */
+    private $_heap = array();
+
+
+    /**
+     * Add element to the queue
+     *
+     * O(log(N)) time
+     *
+     * @param mixed $element
+     */
+    public function put($element)
+    {
+        $nodeId   = count($this->_heap);
+        $parentId = ($nodeId-1) >> 1;   // floor( ($nodeId-1)/2 )
+
+        while ($nodeId != 0  &&  $this->_less($element, $this->_heap[$parentId])) {
+            // Move parent node down
+            $this->_heap[$nodeId] = $this->_heap[$parentId];
+
+            // Move pointer to the next level of tree
+            $nodeId   = $parentId;
+            $parentId = ($nodeId-1) >> 1;   // floor( ($nodeId-1)/2 )
+        }
+
+        // Put new node into the tree
+        $this->_heap[$nodeId] = $element;
+    }
+
+
+    /**
+     * Return least element of the queue
+     *
+     * Constant time
+     *
+     * @return mixed
+     */
+    public function top()
+    {
+        if (count($this->_heap) == 0) {
+            return null;
+        }
+
+        return $this->_heap[0];
+    }
+
+
+    /**
+     * Removes and return least element of the queue
+     *
+     * O(log(N)) time
+     *
+     * @return mixed
+     */
+    public function pop()
+    {
+        if (count($this->_heap) == 0) {
+            return null;
+        }
+
+        $top = $this->_heap[0];
+        $lastId = count($this->_heap) - 1;
+
+        /**
+         * Find appropriate position for last node
+         */
+        $nodeId  = 0;     // Start from a top
+        $childId = 1;     // First child
+
+        // Choose smaller child
+        if ($lastId > 2  &&  $this->_less($this->_heap[2], $this->_heap[1])) {
+            $childId = 2;
+        }
+
+        while ($childId < $lastId  &&
+               $this->_less($this->_heap[$childId], $this->_heap[$lastId])
+          ) {
+            // Move child node up
+            $this->_heap[$nodeId] = $this->_heap[$childId];
+
+            $nodeId  = $childId;               // Go down
+            $childId = ($nodeId << 1) + 1;     // First child
+
+            // Choose smaller child
+            if (($childId+1) < $lastId  &&
+                $this->_less($this->_heap[$childId+1], $this->_heap[$childId])
+               ) {
+                $childId++;
+            }
+        }
+
+        // Move last element to the new position
+        $this->_heap[$nodeId] = $this->_heap[$lastId];
+        unset($this->_heap[$lastId]);
+
+        return $top;
+    }
+
+
+    /**
+     * Clear queue
+     */
+    public function clear()
+    {
+        $this->_heap = array();
+    }
+
+
+    /**
+     * Compare elements
+     *
+     * Returns true, if $el1 is less than $el2; else otherwise
+     *
+     * @param mixed $el1
+     * @param mixed $el2
+     * @return boolean
+     */
+    abstract protected function _less($el1, $el2);
+}
+

search/Zend/Search/Lucene/Proxy.php

@@ -0,0 +1,468 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+/** Zend_Search_Lucene_Interface */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Interface.php';
+
+
+/**
+ * Proxy class intended to be used in userland.
+ *
+ * It tracks, when index object goes out of scope and forces ndex closing
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Proxy implements Zend_Search_Lucene_Interface
+{
+    /**
+     * Index object
+     *
+     * @var Zend_Search_Lucene_Interface
+     */
+    private $_index;
+
+    /**
+     * Object constructor
+     *
+     * @param Zend_Search_Lucene_Interface $index
+     */
+    public function __construct(Zend_Search_Lucene_Interface $index)
+    {
+        $this->_index = $index;
+        $this->_index->addReference();
+    }
+
+    /**
+     * Object destructor
+     */
+    public function __destruct()
+    {
+        if ($this->_index !== null) {
+            // This code is invoked if Zend_Search_Lucene_Interface object constructor throws an exception
+            $this->_index->removeReference();
+        }
+        $this->_index = null;
+    }
+
+    /**
+     * Returns the Zend_Search_Lucene_Storage_Directory instance for this index.
+     *
+     * @return Zend_Search_Lucene_Storage_Directory
+     */
+    public function getDirectory()
+    {
+        return $this->_index->getDirectory();
+    }
+
+    /**
+     * Returns the total number of documents in this index (including deleted documents).
+     *
+     * @return integer
+     */
+    public function count()
+    {
+        return $this->_index->count();
+    }
+
+    /**
+     * Returns one greater than the largest possible document number.
+     * This may be used to, e.g., determine how big to allocate a structure which will have
+     * an element for every document number in an index.
+     *
+     * @return integer
+     */
+    public function maxDoc()
+    {
+        return $this->_index->maxDoc();
+    }
+
+    /**
+     * Returns the total number of non-deleted documents in this index.
+     *
+     * @return integer
+     */
+    public function numDocs()
+    {
+        return $this->_index->numDocs();
+    }
+
+    /**
+     * Checks, that document is deleted
+     *
+     * @param integer $id
+     * @return boolean
+     * @throws Zend_Search_Lucene_Exception    Exception is thrown if $id is out of the range
+     */
+    public function isDeleted($id)
+    {
+        return $this->_index->isDeleted($id);
+    }
+
+    /**
+     * Set default search field.
+     *
+     * Null means, that search is performed through all fields by default
+     *
+     * Default value is null
+     *
+     * @param string $fieldName
+     */
+    public static function setDefaultSearchField($fieldName)
+    {
+        Zend_Search_Lucene::setDefaultSearchField($fieldName);
+    }
+
+    /**
+     * Get default search field.
+     *
+     * Null means, that search is performed through all fields by default
+     *
+     * @return string
+     */
+    public static function getDefaultSearchField()
+    {
+        return Zend_Search_Lucene::getDefaultSearchField();
+    }
+
+    /**
+     * Retrieve index maxBufferedDocs option
+     *
+     * maxBufferedDocs is a minimal number of documents required before
+     * the buffered in-memory documents are written into a new Segment
+     *
+     * Default value is 10
+     *
+     * @return integer
+     */
+    public function getMaxBufferedDocs()
+    {
+        return $this->_index->getMaxBufferedDocs();
+    }
+
+    /**
+     * Set index maxBufferedDocs option
+     *
+     * maxBufferedDocs is a minimal number of documents required before
+     * the buffered in-memory documents are written into a new Segment
+     *
+     * Default value is 10
+     *
+     * @param integer $maxBufferedDocs
+     */
+    public function setMaxBufferedDocs($maxBufferedDocs)
+    {
+        $this->_index->setMaxBufferedDocs($maxBufferedDocs);
+    }
+
+
+    /**
+     * Retrieve index maxMergeDocs option
+     *
+     * maxMergeDocs is a largest number of documents ever merged by addDocument().
+     * Small values (e.g., less than 10,000) are best for interactive indexing,
+     * as this limits the length of pauses while indexing to a few seconds.
+     * Larger values are best for batched indexing and speedier searches.
+     *
+     * Default value is PHP_INT_MAX
+     *
+     * @return integer
+     */
+    public function getMaxMergeDocs()
+    {
+        return $this->_index->getMaxMergeDocs();
+    }
+
+    /**
+     * Set index maxMergeDocs option
+     *
+     * maxMergeDocs is a largest number of documents ever merged by addDocument().
+     * Small values (e.g., less than 10,000) are best for interactive indexing,
+     * as this limits the length of pauses while indexing to a few seconds.
+     * Larger values are best for batched indexing and speedier searches.
+     *
+     * Default value is PHP_INT_MAX
+     *
+     * @param integer $maxMergeDocs
+     */
+    public function setMaxMergeDocs($maxMergeDocs)
+    {
+        $this->_index->setMaxMergeDocs($maxMergeDocs);
+    }
+
+
+    /**
+     * Retrieve index mergeFactor option
+     *
+     * mergeFactor determines how often segment indices are merged by addDocument().
+     * With smaller values, less RAM is used while indexing,
+     * and searches on unoptimized indices are faster,
+     * but indexing speed is slower.
+     * With larger values, more RAM is used during indexing,
+     * and while searches on unoptimized indices are slower,
+     * indexing is faster.
+     * Thus larger values (> 10) are best for batch index creation,
+     * and smaller values (< 10) for indices that are interactively maintained.
+     *
+     * Default value is 10
+     *
+     * @return integer
+     */
+    public function getMergeFactor()
+    {
+        return $this->_index->getMergeFactor();
+    }
+
+    /**
+     * Set index mergeFactor option
+     *
+     * mergeFactor determines how often segment indices are merged by addDocument().
+     * With smaller values, less RAM is used while indexing,
+     * and searches on unoptimized indices are faster,
+     * but indexing speed is slower.
+     * With larger values, more RAM is used during indexing,
+     * and while searches on unoptimized indices are slower,
+     * indexing is faster.
+     * Thus larger values (> 10) are best for batch index creation,
+     * and smaller values (< 10) for indices that are interactively maintained.
+     *
+     * Default value is 10
+     *
+     * @param integer $maxMergeDocs
+     */
+    public function setMergeFactor($mergeFactor)
+    {
+        $this->_index->setMergeFactor($mergeFactor);
+    }
+
+    /**
+     * Performs a query against the index and returns an array
+     * of Zend_Search_Lucene_Search_QueryHit objects.
+     * Input is a string or Zend_Search_Lucene_Search_Query.
+     *
+     * @param mixed $query
+     * @return array Zend_Search_Lucene_Search_QueryHit
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public function find($query)
+    {
+        // actual parameter list
+        $parameters = func_get_args();
+
+        // invoke $this->_index->find() method with specified parameters
+        return call_user_func_array(array(&$this->_index, 'find'), $parameters);
+    }
+
+    /**
+     * Returns a list of all unique field names that exist in this index.
+     *
+     * @param boolean $indexed
+     * @return array
+     */
+    public function getFieldNames($indexed = false)
+    {
+        return $this->_index->getFieldNames($indexed);
+    }
+
+    /**
+     * Returns a Zend_Search_Lucene_Document object for the document
+     * number $id in this index.
+     *
+     * @param integer|Zend_Search_Lucene_Search_QueryHit $id
+     * @return Zend_Search_Lucene_Document
+     */
+    public function getDocument($id)
+    {
+        return $this->_index->getDocument($id);
+    }
+
+    /**
+     * Returns true if index contain documents with specified term.
+     *
+     * Is used for query optimization.
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @return boolean
+     */
+    public function hasTerm(Zend_Search_Lucene_Index_Term $term)
+    {
+        return $this->_index->hasTerm($term);
+    }
+
+    /**
+     * Returns IDs of all the documents containing term.
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @return array
+     */
+    public function termDocs(Zend_Search_Lucene_Index_Term $term)
+    {
+        return $this->_index->termDocs($term);
+    }
+
+    /**
+     * Returns an array of all term freqs.
+     * Return array structure: array( docId => freq, ...)
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @return integer
+     */
+    public function termFreqs(Zend_Search_Lucene_Index_Term $term)
+    {
+        return $this->_index->termFreqs($term);
+    }
+
+    /**
+     * Returns an array of all term positions in the documents.
+     * Return array structure: array( docId => array( pos1, pos2, ...), ...)
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @return array
+     */
+    public function termPositions(Zend_Search_Lucene_Index_Term $term)
+    {
+        return $this->_index->termPositions($term);
+    }
+
+    /**
+     * Returns the number of documents in this index containing the $term.
+     *
+     * @param Zend_Search_Lucene_Index_Term $term
+     * @return integer
+     */
+    public function docFreq(Zend_Search_Lucene_Index_Term $term)
+    {
+        return $this->_index->docFreq($term);
+    }
+
+    /**
+     * Retrive similarity used by index reader
+     *
+     * @return Zend_Search_Lucene_Search_Similarity
+     */
+    public function getSimilarity()
+    {
+        return $this->_index->getSimilarity();
+    }
+
+    /**
+     * Returns a normalization factor for "field, document" pair.
+     *
+     * @param integer $id
+     * @param string $fieldName
+     * @return float
+     */
+    public function norm($id, $fieldName)
+    {
+        return $this->_index->norm($id, $fieldName);
+    }
+
+    /**
+     * Returns true if any documents have been deleted from this index.
+     *
+     * @return boolean
+     */
+    public function hasDeletions()
+    {
+        return $this->_index->hasDeletions();
+    }
+
+    /**
+     * Deletes a document from the index.
+     * $id is an internal document id
+     *
+     * @param integer|Zend_Search_Lucene_Search_QueryHit $id
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public function delete($id)
+    {
+        return $this->_index->delete($id);
+    }
+
+    /**
+     * Adds a document to this index.
+     *
+     * @param Zend_Search_Lucene_Document $document
+     */
+    public function addDocument(Zend_Search_Lucene_Document $document)
+    {
+        $this->_index->addDocument($document);
+    }
+
+    /**
+     * Commit changes resulting from delete() or undeleteAll() operations.
+     */
+    public function commit()
+    {
+        $this->_index->commit();
+    }
+
+    /**
+     * Optimize index.
+     *
+     * Merges all segments into one
+     */
+    public function optimize()
+    {
+        $this->_index->optimize();
+    }
+
+    /**
+     * Returns an array of all terms in this index.
+     *
+     * @return array
+     */
+    public function terms()
+    {
+        return $this->_index->terms();
+    }
+
+    /**
+     * Undeletes all documents currently marked as deleted in this index.
+     */
+    public function undeleteAll()
+    {
+        return $this->_index->undeleteAll();
+    }
+
+    /**
+     * Add reference to the index object
+     *
+     * @internal
+     */
+    public function addReference()
+    {
+        return $this->_index->addReference();
+    }
+
+    /**
+     * Remove reference from the index object
+     *
+     * When reference count becomes zero, index is closed and resources are cleaned up
+     *
+     * @internal
+     */
+    public function removeReference()
+    {
+        return $this->_index->removeReference();
+    }
+}

search/Zend/Search/Lucene/Search/BooleanExpressionRecognizer.php

@@ -0,0 +1,280 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_FSM */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/FSM.php';
+
+/** Zend_Search_Lucene_Search_QueryToken */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryToken.php';
+
+/** Zend_Search_Lucene_Search_QueryParser */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParser.php';
+
+
+/** Zend_Search_Lucene_Exception */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Search_BooleanExpressionRecognizer extends Zend_Search_Lucene_FSM
+{
+    /** State Machine states */
+    const ST_START           = 0;
+    const ST_LITERAL         = 1;
+    const ST_NOT_OPERATOR    = 2;
+    const ST_AND_OPERATOR    = 3;
+    const ST_OR_OPERATOR     = 4;
+
+    /** Input symbols */
+    const IN_LITERAL         = 0;
+    const IN_NOT_OPERATOR    = 1;
+    const IN_AND_OPERATOR    = 2;
+    const IN_OR_OPERATOR     = 3;
+
+
+    /**
+     * NOT operator signal
+     *
+     * @var boolean
+     */
+    private $_negativeLiteral = false;
+
+    /**
+     * Current literal
+     *
+     * @var mixed
+     */
+    private $_literal;
+
+
+    /**
+     * Set of boolean query conjunctions
+     *
+     * Each conjunction is an array of conjunction elements
+     * Each conjunction element is presented with two-elements array:
+     * array(<literal>, <is_negative>)
+     *
+     * So, it has a structure:
+     * array( array( array(<literal>, <is_negative>), // first literal of first conjuction
+     *               array(<literal>, <is_negative>), // second literal of first conjuction
+     *               ...
+     *               array(<literal>, <is_negative>)
+     *             ), // end of first conjuction
+     *        array( array(<literal>, <is_negative>), // first literal of second conjuction
+     *               array(<literal>, <is_negative>), // second literal of second conjuction
+     *               ...
+     *               array(<literal>, <is_negative>)
+     *             ), // end of second conjuction
+     *        ...
+     *      ) // end of structure
+     *
+     * @var array
+     */
+    private $_conjunctions = array();
+
+    /**
+     * Current conjuction
+     *
+     * @var array
+     */
+    private $_currentConjunction = array();
+
+
+    /**
+     * Object constructor
+     */
+    public function __construct()
+    {
+        parent::__construct( array(self::ST_START,
+                                   self::ST_LITERAL,
+                                   self::ST_NOT_OPERATOR,
+                                   self::ST_AND_OPERATOR,
+                                   self::ST_OR_OPERATOR),
+                             array(self::IN_LITERAL,
+                                   self::IN_NOT_OPERATOR,
+                                   self::IN_AND_OPERATOR,
+                                   self::IN_OR_OPERATOR));
+
+        $emptyOperatorAction    = new Zend_Search_Lucene_FSMAction($this, 'emptyOperatorAction');
+        $emptyNotOperatorAction = new Zend_Search_Lucene_FSMAction($this, 'emptyNotOperatorAction');
+
+        $this->addRules(array( array(self::ST_START,        self::IN_LITERAL,        self::ST_LITERAL),
+                               array(self::ST_START,        self::IN_NOT_OPERATOR,   self::ST_NOT_OPERATOR),
+
+                               array(self::ST_LITERAL,      self::IN_AND_OPERATOR,   self::ST_AND_OPERATOR),
+                               array(self::ST_LITERAL,      self::IN_OR_OPERATOR,    self::ST_OR_OPERATOR),
+                               array(self::ST_LITERAL,      self::IN_LITERAL,        self::ST_LITERAL,      $emptyOperatorAction),
+                               array(self::ST_LITERAL,      self::IN_NOT_OPERATOR,   self::ST_NOT_OPERATOR, $emptyNotOperatorAction),
+
+                               array(self::ST_NOT_OPERATOR, self::IN_LITERAL,        self::ST_LITERAL),
+
+                               array(self::ST_AND_OPERATOR, self::IN_LITERAL,        self::ST_LITERAL),
+                               array(self::ST_AND_OPERATOR, self::IN_NOT_OPERATOR,   self::ST_NOT_OPERATOR),
+
+                               array(self::ST_OR_OPERATOR,  self::IN_LITERAL,        self::ST_LITERAL),
+                               array(self::ST_OR_OPERATOR,  self::IN_NOT_OPERATOR,   self::ST_NOT_OPERATOR),
+                             ));
+
+        $notOperatorAction     = new Zend_Search_Lucene_FSMAction($this, 'notOperatorAction');
+        $orOperatorAction      = new Zend_Search_Lucene_FSMAction($this, 'orOperatorAction');
+        $literalAction         = new Zend_Search_Lucene_FSMAction($this, 'literalAction');
+
+
+        $this->addEntryAction(self::ST_NOT_OPERATOR, $notOperatorAction);
+        $this->addEntryAction(self::ST_OR_OPERATOR,  $orOperatorAction);
+        $this->addEntryAction(self::ST_LITERAL,      $literalAction);
+    }
+
+
+    /**
+     * Process next operator.
+     *
+     * Operators are defined by class constants: IN_AND_OPERATOR, IN_OR_OPERATOR and IN_NOT_OPERATOR
+     *
+     * @param integer $operator
+     */
+    public function processOperator($operator)
+    {
+        $this->process($operator);
+    }
+
+    /**
+     * Process expression literal.
+     *
+     * @param integer $operator
+     */
+    public function processLiteral($literal)
+    {
+        $this->_literal = $literal;
+
+        $this->process(self::IN_LITERAL);
+    }
+
+    /**
+     * Finish an expression and return result
+     *
+     * Result is a set of boolean query conjunctions
+     *
+     * Each conjunction is an array of conjunction elements
+     * Each conjunction element is presented with two-elements array:
+     * array(<literal>, <is_negative>)
+     *
+     * So, it has a structure:
+     * array( array( array(<literal>, <is_negative>), // first literal of first conjuction
+     *               array(<literal>, <is_negative>), // second literal of first conjuction
+     *               ...
+     *               array(<literal>, <is_negative>)
+     *             ), // end of first conjuction
+     *        array( array(<literal>, <is_negative>), // first literal of second conjuction
+     *               array(<literal>, <is_negative>), // second literal of second conjuction
+     *               ...
+     *               array(<literal>, <is_negative>)
+     *             ), // end of second conjuction
+     *        ...
+     *      ) // end of structure
+     *
+     * @return array
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public function finishExpression()
+    {
+        if ($this->getState() != self::ST_LITERAL) {
+            throw new Zend_Search_Lucene_Exception('Literal expected.');
+        }
+
+        $this->_conjunctions[] = $this->_currentConjunction;
+
+        return $this->_conjunctions;
+    }
+
+
+
+    /*********************************************************************
+     * Actions implementation
+     *********************************************************************/
+
+    /**
+     * default (omitted) operator processing
+     */
+    public function emptyOperatorAction()
+    {
+        if (Zend_Search_Lucene_Search_QueryParser::getDefaultOperator() == Zend_Search_Lucene_Search_QueryParser::B_AND) {
+            // Do nothing
+        } else {
+            $this->orOperatorAction();
+        }
+
+        // Process literal
+        $this->literalAction();
+    }
+
+    /**
+     * default (omitted) + NOT operator processing
+     */
+    public function emptyNotOperatorAction()
+    {
+        if (Zend_Search_Lucene_Search_QueryParser::getDefaultOperator() == Zend_Search_Lucene_Search_QueryParser::B_AND) {
+            // Do nothing
+        } else {
+            $this->orOperatorAction();
+        }
+
+        // Process NOT operator
+        $this->notOperatorAction();
+    }
+
+
+    /**
+     * NOT operator processing
+     */
+    public function notOperatorAction()
+    {
+        $this->_negativeLiteral = true;
+    }
+
+    /**
+     * OR operator processing
+     * Close current conjunction
+     */
+    public function orOperatorAction()
+    {
+        $this->_conjunctions[]     = $this->_currentConjunction;
+        $this->_currentConjunction = array();
+    }
+
+    /**
+     * Literal processing
+     */
+    public function literalAction()
+    {
+        // Add literal to the current conjunction
+        $this->_currentConjunction[] = array($this->_literal, !$this->_negativeLiteral);
+
+        // Switch off negative signal
+        $this->_negativeLiteral = false;
+    }
+}

search/Zend/Search/Lucene/Search/Query.php

@@ -15,16 +15,19 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
+/** Zend_Search_Lucene_Document_Html */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Document/Html.php';
+
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 abstract class Zend_Search_Lucene_Search_Query
@@ -35,14 +38,31 @@ abstract class Zend_Search_Lucene_Search_Query
      *
      * @var float
      */
-    private $_boost = 1.0;
+    private $_boost = 1;
 
     /**
      * Query weight
      *
      * @var Zend_Search_Lucene_Search_Weight
      */
-    protected $_weight;
+    protected $_weight = null;
+
+    /**
+     * Current highlight color
+     *
+     * @var integer
+     */
+    private $_currentColorIndex = 0;
+
+    /**
+     * List of colors for text highlighting
+     *
+     * @var array
+     */
+    private $_highlightColors = array('#66ffff', '#ff66ff', '#ffff66',
+                                      '#ff8888', '#88ff88', '#8888ff',
+                                      '#88dddd', '#dd88dd', '#dddd88',
+                                      '#aaddff', '#aaffdd', '#ddaaff', '#ddffaa', '#ffaadd', '#ffddaa');
 
 
     /**
@@ -71,30 +91,133 @@ abstract class Zend_Search_Lucene_Search_Query
      * Score specified document
      *
      * @param integer $docId
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      * @return float
      */
-    abstract public function score($docId, $reader);
+    abstract public function score($docId, Zend_Search_Lucene_Interface $reader);
+
+    /**
+     * Get document ids likely matching the query
+     *
+     * It's an array with document ids as keys (performance considerations)
+     *
+     * @return array
+     */
+    abstract public function matchedDocs();
+
+    /**
+     * Execute query in context of index reader
+     * It also initializes necessary internal structures
+     *
+     * Query specific implementation
+     *
+     * @param Zend_Search_Lucene_Interface $reader
+     */
+    abstract public function execute(Zend_Search_Lucene_Interface $reader);
 
     /**
      * Constructs an appropriate Weight implementation for this query.
      *
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      * @return Zend_Search_Lucene_Search_Weight
      */
-    abstract protected function _createWeight($reader);
+    abstract public function createWeight(Zend_Search_Lucene_Interface $reader);
 
     /**
-     * Constructs an initializes a Weight for a query.
+     * Constructs an initializes a Weight for a _top-level_query_.
      *
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      */
-    protected function _initWeight($reader)
+    protected function _initWeight(Zend_Search_Lucene_Interface $reader)
     {
-        $this->_weight = $this->_createWeight($reader);
+        // Check, that it's a top-level query and query weight is not initialized yet.
+        if ($this->_weight !== null) {
+            return $this->_weight;
+        }
+
+        $this->createWeight($reader);
         $sum = $this->_weight->sumOfSquaredWeights();
         $queryNorm = $reader->getSimilarity()->queryNorm($sum);
         $this->_weight->normalize($queryNorm);
     }
 
-}
+    /**
+     * Re-write query into primitive queries in the context of specified index
+     *
+     * @param Zend_Search_Lucene_Interface $index
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    abstract public function rewrite(Zend_Search_Lucene_Interface $index);
+
+    /**
+     * Optimize query in the context of specified index
+     *
+     * @param Zend_Search_Lucene_Interface $index
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    abstract public function optimize(Zend_Search_Lucene_Interface $index);
+
+    /**
+     * Reset query, so it can be reused within other queries or
+     * with other indeces
+     */
+    public function reset()
+    {
+        $this->_weight = null;
+    }
+
+
+    /**
+     * Print a query
+     *
+     * @return string
+     */
+    abstract public function __toString();
+
+    /**
+     * Return query terms
+     *
+     * @return array
+     */
+    abstract public function getQueryTerms();
+
+    /**
+     * Get highlight color and shift to next
+     *
+     * @param integer &$colorIndex
+     * @return string
+     */
+    protected function _getHighlightColor(&$colorIndex)
+    {
+        $color = $this->_highlightColors[$colorIndex++];
+
+        $colorIndex %= count($this->_highlightColors);
+
+        return $color;
+    }
+
+    /**
+     * Highlight query terms
+     *
+     * @param integer &$colorIndex
+     * @param Zend_Search_Lucene_Document_Html $doc
+     */
+    abstract public function highlightMatchesDOM(Zend_Search_Lucene_Document_Html $doc, &$colorIndex);
+
+    /**
+     * Highlight matches in $inputHTML
+     *
+     * @param string $inputHTML
+     * @return string
+     */
+    public function highlightMatches($inputHTML)
+    {
+        $doc = Zend_Search_Lucene_Document_Html::loadHTML($inputHTML);
+
+        $colorIndex = 0;
+        $this->highlightMatchesDOM($doc, $colorIndex);
+
+        return $doc->getHTML();
+    }
+}
+

search/Zend/Search/Lucene/Search/Query/Boolean.php

@@ -0,0 +1,715 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Search_Query */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query.php';
+
+/** Zend_Search_Lucene_Search_Weight_Boolean */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/Boolean.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Search_Query_Boolean extends Zend_Search_Lucene_Search_Query
+{
+
+    /**
+     * Subqueries
+     * Array of Zend_Search_Lucene_Query
+     *
+     * @var array
+     */
+    private $_subqueries = array();
+
+    /**
+     * Subqueries signs.
+     * If true then subquery is required.
+     * If false then subquery is prohibited.
+     * If null then subquery is neither prohibited, nor required
+     *
+     * If array is null then all subqueries are required
+     *
+     * @var array
+     */
+    private $_signs = array();
+
+    /**
+     * Result vector.
+     *
+     * @var array
+     */
+    private $_resVector = null;
+
+    /**
+     * A score factor based on the fraction of all query subqueries
+     * that a document contains.
+     * float for conjunction queries
+     * array of float for non conjunction queries
+     *
+     * @var mixed
+     */
+    private $_coord = null;
+
+
+    /**
+     * Class constructor.  Create a new Boolean query object.
+     *
+     * if $signs array is omitted then all subqueries are required
+     * it differs from addSubquery() behavior, but should never be used
+     *
+     * @param array $subqueries    Array of Zend_Search_Search_Query objects
+     * @param array $signs    Array of signs.  Sign is boolean|null.
+     * @return void
+     */
+    public function __construct($subqueries = null, $signs = null)
+    {
+        if (is_array($subqueries)) {
+            $this->_subqueries = $subqueries;
+
+            $this->_signs = null;
+            // Check if all subqueries are required
+            if (is_array($signs)) {
+                foreach ($signs as $sign ) {
+                    if ($sign !== true) {
+                        $this->_signs = $signs;
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+
+    /**
+     * Add a $subquery (Zend_Search_Lucene_Query) to this query.
+     *
+     * The sign is specified as:
+     *     TRUE  - subquery is required
+     *     FALSE - subquery is prohibited
+     *     NULL  - subquery is neither prohibited, nor required
+     *
+     * @param  Zend_Search_Lucene_Search_Query $subquery
+     * @param  boolean|null $sign
+     * @return void
+     */
+    public function addSubquery(Zend_Search_Lucene_Search_Query $subquery, $sign=null) {
+        if ($sign !== true || $this->_signs !== null) {       // Skip, if all subqueries are required
+            if ($this->_signs === null) {                     // Check, If all previous subqueries are required
+                foreach ($this->_subqueries as $prevSubquery) {
+                    $this->_signs[] = true;
+                }
+            }
+            $this->_signs[] = $sign;
+        }
+
+        $this->_subqueries[] = $subquery;
+    }
+
+    /**
+     * Re-write queries into primitive queries
+     *
+     * @param Zend_Search_Lucene_Interface $index
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    public function rewrite(Zend_Search_Lucene_Interface $index)
+    {
+        $query = new Zend_Search_Lucene_Search_Query_Boolean();
+        $query->setBoost($this->getBoost());
+
+        foreach ($this->_subqueries as $subqueryId => $subquery) {
+            $query->addSubquery($subquery->rewrite($index),
+                                ($this->_signs === null)?  true : $this->_signs[$subqueryId]);
+        }
+
+        return $query;
+    }
+
+    /**
+     * Optimize query in the context of specified index
+     *
+     * @param Zend_Search_Lucene_Interface $index
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    public function optimize(Zend_Search_Lucene_Interface $index)
+    {
+        $subqueries = array();
+        $signs      = array();
+
+        // Optimize all subqueries
+        foreach ($this->_subqueries as $id => $subquery) {
+            $subqueries[] = $subquery->optimize($index);
+            $signs[]      = ($this->_signs === null)? true : $this->_signs[$id];
+        }
+
+        // Check for empty subqueries
+        foreach ($subqueries as $id => $subquery) {
+            if ($subquery instanceof Zend_Search_Lucene_Search_Query_Empty) {
+                if ($signs[$id] === true) {
+                    // Matching is required, but is actually empty
+                    return new Zend_Search_Lucene_Search_Query_Empty();
+                } else {
+                    // Matching is optional or prohibited, but is empty
+                    // Remove it from subqueries and signs list
+                    unset($subqueries[$id]);
+                    unset($signs[$id]);
+                }
+            }
+        }
+
+
+        // Check if all non-empty subqueries are prohibited
+        $allProhibited = true;
+        foreach ($signs as $sign) {
+            if ($sign !== false) {
+                $allProhibited = false;
+                break;
+            }
+        }
+        if ($allProhibited) {
+            return new Zend_Search_Lucene_Search_Query_Empty();
+        }
+
+
+        // Check, if reduced subqueries list has only one entry
+        if (count($subqueries) == 1) {
+            // It's a query with only one required or optional clause
+            // (it's already checked, that it's not a prohibited clause)
+
+            if ($this->getBoost() == 1) {
+                return reset($subqueries);
+            }
+
+            $optimizedQuery = clone reset($subqueries);
+            $optimizedQuery->setBoost($optimizedQuery->getBoost()*$this->getBoost());
+
+            return $optimizedQuery;
+        }
+
+
+        // Check, if reduced subqueries list is empty
+        if (count($subqueries) == 0) {
+            return new Zend_Search_Lucene_Search_Query_Empty();
+        }
+
+
+        // Prepare first candidate for optimized query
+        $optimizedQuery = new Zend_Search_Lucene_Search_Query_Boolean($subqueries, $signs);
+        $optimizedQuery->setBoost($this->getBoost());
+
+
+        $terms        = array();
+        $tsigns       = array();
+        $boostFactors = array();
+
+        // Try to decompose term and multi-term subqueries
+        foreach ($subqueries as $id => $subquery) {
+            if ($subquery instanceof Zend_Search_Lucene_Search_Query_Term) {
+                $terms[]        = $subquery->getTerm();
+                $tsigns[]       = $signs[$id];
+                $boostFactors[] = $subquery->getBoost();
+
+                // remove subquery from a subqueries list
+                unset($subqueries[$id]);
+                unset($signs[$id]);
+           } else if ($subquery instanceof Zend_Search_Lucene_Search_Query_MultiTerm) {
+                $subTerms = $subquery->getTerms();
+                $subSigns = $subquery->getSigns();
+
+                if ($signs[$id] === true) {
+                    // It's a required multi-term subquery.
+                    // Something like '... +(+term1 -term2 term3 ...) ...'
+
+                    // Multi-term required subquery can be decomposed only if it contains
+                    // required terms and doesn't contain prohibited terms:
+                    // ... +(+term1 term2 ...) ... => ... +term1 term2 ...
+                    //
+                    // Check this
+                    $hasRequired   = false;
+                    $hasProhibited = false;
+                    if ($subSigns === null) {
+                        // All subterms are required
+                        $hasRequired = true;
+                    } else {
+                        foreach ($subSigns as $sign) {
+                            if ($sign === true) {
+                                $hasRequired   = true;
+                            } else if ($sign === false) {
+                                $hasProhibited = true;
+                                break;
+                            }
+                        }
+                    }
+                    // Continue if subquery has prohibited terms or doesn't have required terms
+                    if ($hasProhibited  ||  !$hasRequired) {
+                        continue;
+                    }
+
+                    foreach ($subTerms as $termId => $term) {
+                        $terms[]        = $term;
+                        $tsigns[]       = ($subSigns === null)? true : $subSigns[$termId];
+                        $boostFactors[] = $subquery->getBoost();
+                    }
+
+                    // remove subquery from a subqueries list
+                    unset($subqueries[$id]);
+                    unset($signs[$id]);
+
+                } else { // $signs[$id] === null  ||  $signs[$id] === false
+                    // It's an optional or prohibited multi-term subquery.
+                    // Something like '... (+term1 -term2 term3 ...) ...'
+                    // or
+                    // something like '... -(+term1 -term2 term3 ...) ...'
+
+                    // Multi-term optional and required subqueries can be decomposed
+                    // only if all terms are optional.
+                    //
+                    // Check if all terms are optional.
+                    $onlyOptional = true;
+                    if ($subSigns === null) {
+                        // All subterms are required
+                        $onlyOptional = false;
+                    } else {
+                        foreach ($subSigns as $sign) {
+                            if ($sign !== null) {
+                                $onlyOptional = false;
+                                break;
+                            }
+                        }
+                    }
+
+                    // Continue if non-optional terms are presented in this multi-term subquery
+                    if (!$onlyOptional) {
+                        continue;
+                    }
+
+                    foreach ($subTerms as $termId => $term) {
+                        $terms[]  = $term;
+                        $tsigns[] = ($signs[$id] === null)? null  /* optional */ :
+                                                            false /* prohibited */;
+                        $boostFactors[] = $subquery->getBoost();
+                    }
+
+                    // remove subquery from a subqueries list
+                    unset($subqueries[$id]);
+                    unset($signs[$id]);
+                }
+            }
+        }
+
+
+        // Check, if there are no decomposed subqueries
+        if (count($terms) == 0 ) {
+            // return prepared candidate
+            return $optimizedQuery;
+        }
+
+
+        // Check, if all subqueries have been decomposed and all terms has the same boost factor
+        if (count($subqueries) == 0  &&  count(array_unique($boostFactors)) == 1) {
+            $optimizedQuery = new Zend_Search_Lucene_Search_Query_MultiTerm($terms, $tsigns);
+            $optimizedQuery->setBoost(reset($boostFactors)*$this->getBoost());
+
+            return $optimizedQuery;
+        }
+
+
+        // This boolean query can't be transformed to Term/MultiTerm query and still contains
+        // several subqueries
+
+        // Separate prohibited terms
+        $prohibitedTerms        = array();
+        foreach ($terms as $id => $term) {
+            if ($tsigns[$id] === false) {
+                $prohibitedTerms[]        = $term;
+
+                unset($terms[$id]);
+                unset($tsigns[$id]);
+                unset($boostFactors[$id]);
+            }
+        }
+
+        if (count($terms) == 1) {
+            $clause = new Zend_Search_Lucene_Search_Query_Term(reset($terms));
+            $clause->setBoost(reset($boostFactors));
+
+            $subqueries[] = $clause;
+            $signs[]      = reset($tsigns);
+
+            // Clear terms list
+            $terms = array();
+        } else if (count($terms) > 1  &&  count(array_unique($boostFactors)) == 1) {
+            $clause = new Zend_Search_Lucene_Search_Query_MultiTerm($terms, $tsigns);
+            $clause->setBoost(reset($boostFactors));
+
+            $subqueries[] = $clause;
+            // Clause sign is 'required' if clause contains required terms. 'Optional' otherwise.
+            $signs[]      = (in_array(true, $tsigns))? true : null;
+
+            // Clear terms list
+            $terms = array();
+        }
+
+        if (count($prohibitedTerms) == 1) {
+            // (boost factors are not significant for prohibited clauses)
+            $subqueries[] = new Zend_Search_Lucene_Search_Query_Term(reset($prohibitedTerms));
+            $signs[]      = false;
+
+            // Clear prohibited terms list
+            $prohibitedTerms = array();
+        } else if (count($prohibitedTerms) > 1) {
+            // prepare signs array
+            $prohibitedSigns = array();
+            foreach ($prohibitedTerms as $id => $term) {
+                // all prohibited term are grouped as optional into multi-term query
+                $prohibitedSigns[$id] = null;
+            }
+
+            // (boost factors are not significant for prohibited clauses)
+            $subqueries[] = new Zend_Search_Lucene_Search_Query_MultiTerm($prohibitedTerms, $prohibitedSigns);
+            // Clause sign is 'prohibited'
+            $signs[]      = false;
+
+            // Clear terms list
+            $prohibitedTerms = array();
+        }
+
+        /** @todo Group terms with the same boost factors together */
+
+        // Check, that all terms are processed
+        // Replace candidate for optimized query
+        if (count($terms) == 0  &&  count($prohibitedTerms) == 0) {
+            $optimizedQuery = new Zend_Search_Lucene_Search_Query_Boolean($subqueries, $signs);
+            $optimizedQuery->setBoost($this->getBoost());
+        }
+
+        return $optimizedQuery;
+    }
+
+    /**
+     * Returns subqueries
+     *
+     * @return array
+     */
+    public function getSubqueries()
+    {
+        return $this->_subqueries;
+    }
+
+
+    /**
+     * Return subqueries signs
+     *
+     * @return array
+     */
+    public function getSigns()
+    {
+        return $this->_signs;
+    }
+
+
+    /**
+     * Constructs an appropriate Weight implementation for this query.
+     *
+     * @param Zend_Search_Lucene_Interface $reader
+     * @return Zend_Search_Lucene_Search_Weight
+     */
+    public function createWeight(Zend_Search_Lucene_Interface $reader)
+    {
+        $this->_weight = new Zend_Search_Lucene_Search_Weight_Boolean($this, $reader);
+        return $this->_weight;
+    }
+
+
+    /**
+     * Calculate result vector for Conjunction query
+     * (like '<subquery1> AND <subquery2> AND <subquery3>')
+     */
+    private function _calculateConjunctionResult()
+    {
+        $this->_resVector = null;
+
+        if (count($this->_subqueries) == 0) {
+            $this->_resVector = array();
+        }
+
+        foreach ($this->_subqueries as $subquery) {
+            if($this->_resVector === null) {
+                $this->_resVector = $subquery->matchedDocs();
+            } else {
+                $this->_resVector = array_intersect_key($this->_resVector, $subquery->matchedDocs());
+            }
+
+            if (count($this->_resVector) == 0) {
+                // Empty result set, we don't need to check other terms
+                break;
+            }
+        }
+
+        ksort($this->_resVector, SORT_NUMERIC);
+    }
+
+
+    /**
+     * Calculate result vector for non Conjunction query
+     * (like '<subquery1> AND <subquery2> AND NOT <subquery3> OR <subquery4>')
+     */
+    private function _calculateNonConjunctionResult()
+    {
+        $required   = null;
+        $optional   = array();
+
+        foreach ($this->_subqueries as $subqueryId => $subquery) {
+            $docs = $subquery->matchedDocs();
+
+            if ($this->_signs[$subqueryId] === true) {
+                // required
+                if ($required !== null) {
+                    // array intersection
+                    $required = array_intersect_key($required, $docs);
+                } else {
+                    $required = $docs;
+                }
+            } elseif ($this->_signs[$subqueryId] === false) {
+                // prohibited
+                // Do nothing. matchedDocs() may include non-matching id's
+            } else {
+                // neither required, nor prohibited
+                // array union
+                $optional += $docs;
+            }
+        }
+
+        if ($required !== null) {
+            $this->_resVector = &$required;
+        } else {
+            $this->_resVector = &$optional;
+        }
+
+        ksort($this->_resVector, SORT_NUMERIC);
+    }
+
+
+    /**
+     * Score calculator for conjunction queries (all subqueries are required)
+     *
+     * @param integer $docId
+     * @param Zend_Search_Lucene_Interface $reader
+     * @return float
+     */
+    public function _conjunctionScore($docId, Zend_Search_Lucene_Interface $reader)
+    {
+        if ($this->_coord === null) {
+            $this->_coord = $reader->getSimilarity()->coord(count($this->_subqueries),
+                                                            count($this->_subqueries) );
+        }
+
+        $score = 0;
+
+        foreach ($this->_subqueries as $subquery) {
+            $subscore = $subquery->score($docId, $reader);
+
+            if ($subscore == 0) {
+                return 0;
+            }
+
+            $score += $subquery->score($docId, $reader) * $this->_coord;
+        }
+
+        return $score * $this->_coord * $this->getBoost();
+    }
+
+
+    /**
+     * Score calculator for non conjunction queries (not all subqueries are required)
+     *
+     * @param integer $docId
+     * @param Zend_Search_Lucene_Interface $reader
+     * @return float
+     */
+    public function _nonConjunctionScore($docId, Zend_Search_Lucene_Interface $reader)
+    {
+        if ($this->_coord === null) {
+            $this->_coord = array();
+
+            $maxCoord = 0;
+            foreach ($this->_signs as $sign) {
+                if ($sign !== false /* not prohibited */) {
+                    $maxCoord++;
+                }
+            }
+
+            for ($count = 0; $count <= $maxCoord; $count++) {
+                $this->_coord[$count] = $reader->getSimilarity()->coord($count, $maxCoord);
+            }
+        }
+
+        $score = 0;
+        $matchedSubqueries = 0;
+        foreach ($this->_subqueries as $subqueryId => $subquery) {
+            $subscore = $subquery->score($docId, $reader);
+
+            // Prohibited
+            if ($this->_signs[$subqueryId] === false && $subscore != 0) {
+                return 0;
+            }
+
+            // is required, but doen't match
+            if ($this->_signs[$subqueryId] === true &&  $subscore == 0) {
+                return 0;
+            }
+
+            if ($subscore != 0) {
+                $matchedSubqueries++;
+                $score += $subscore;
+            }
+        }
+
+        return $score * $this->_coord[$matchedSubqueries] * $this->getBoost();
+    }
+
+    /**
+     * Execute query in context of index reader
+     * It also initializes necessary internal structures
+     *
+     * @param Zend_Search_Lucene_Interface $reader
+     */
+    public function execute(Zend_Search_Lucene_Interface $reader)
+    {
+        // Initialize weight if it's not done yet
+        $this->_initWeight($reader);
+
+        foreach ($this->_subqueries as $subquery) {
+            $subquery->execute($reader);
+        }
+
+        if ($this->_signs === null) {
+            $this->_calculateConjunctionResult();
+        } else {
+            $this->_calculateNonConjunctionResult();
+        }
+    }
+
+
+
+    /**
+     * Get document ids likely matching the query
+     *
+     * It's an array with document ids as keys (performance considerations)
+     *
+     * @return array
+     */
+    public function matchedDocs()
+    {
+        return $this->_resVector;
+    }
+
+    /**
+     * Score specified document
+     *
+     * @param integer $docId
+     * @param Zend_Search_Lucene_Interface $reader
+     * @return float
+     */
+    public function score($docId, Zend_Search_Lucene_Interface $reader)
+    {
+        if (isset($this->_resVector[$docId])) {
+            if ($this->_signs === null) {
+                return $this->_conjunctionScore($docId, $reader);
+            } else {
+                return $this->_nonConjunctionScore($docId, $reader);
+            }
+        } else {
+            return 0;
+        }
+    }
+
+    /**
+     * Return query terms
+     *
+     * @return array
+     */
+    public function getQueryTerms()
+    {
+        $terms = array();
+
+        foreach ($this->_subqueries as $id => $subquery) {
+            if ($this->_signs === null  ||  $this->_signs[$id] !== false) {
+                $terms = array_merge($terms, $subquery->getQueryTerms());
+            }
+        }
+
+        return $terms;
+    }
+
+    /**
+     * Highlight query terms
+     *
+     * @param integer &$colorIndex
+     * @param Zend_Search_Lucene_Document_Html $doc
+     */
+    public function highlightMatchesDOM(Zend_Search_Lucene_Document_Html $doc, &$colorIndex)
+    {
+        foreach ($this->_subqueries as $id => $subquery) {
+            if ($this->_signs === null  ||  $this->_signs[$id] !== false) {
+                $subquery->highlightMatchesDOM($doc, $colorIndex);
+            }
+        }
+    }
+
+    /**
+     * Print a query
+     *
+     * @return string
+     */
+    public function __toString()
+    {
+        // It's used only for query visualisation, so we don't care about characters escaping
+
+        $query = '';
+
+        foreach ($this->_subqueries as $id => $subquery) {
+            if ($id != 0) {
+                $query .= ' ';
+            }
+
+            if ($this->_signs === null || $this->_signs[$id] === true) {
+                $query .= '+';
+            } else if ($this->_signs[$id] === false) {
+                $query .= '-';
+            }
+
+            $query .= '(' . $subquery->__toString() . ')';
+
+            if ($subquery->getBoost() != 1) {
+                $query .= '^' . $subquery->getBoost();
+            }
+        }
+
+        return $query;
+    }
+}
+

search/Zend/Search/Lucene/Search/Query/Empty.php

@@ -0,0 +1,139 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Search_Query */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query.php';
+
+/** Zend_Search_Lucene_Search_Weight_Empty */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/Empty.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Search_Query_Empty extends Zend_Search_Lucene_Search_Query
+{
+    /**
+     * Re-write query into primitive queries in the context of specified index
+     *
+     * @param Zend_Search_Lucene_Interface $index
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    public function rewrite(Zend_Search_Lucene_Interface $index)
+    {
+        return $this;
+    }
+
+    /**
+     * Optimize query in the context of specified index
+     *
+     * @param Zend_Search_Lucene_Interface $index
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    public function optimize(Zend_Search_Lucene_Interface $index)
+    {
+        // "Empty" query is a primitive query and don't need to be optimized
+        return $this;
+    }
+
+    /**
+     * Constructs an appropriate Weight implementation for this query.
+     *
+     * @param Zend_Search_Lucene_Interface $reader
+     * @return Zend_Search_Lucene_Search_Weight
+     */
+    public function createWeight(Zend_Search_Lucene_Interface $reader)
+    {
+        return new Zend_Search_Lucene_Search_Weight_Empty();
+    }
+
+    /**
+     * Execute query in context of index reader
+     * It also initializes necessary internal structures
+     *
+     * @param Zend_Search_Lucene_Interface $reader
+     */
+    public function execute(Zend_Search_Lucene_Interface $reader)
+    {
+        // Do nothing
+    }
+
+    /**
+     * Get document ids likely matching the query
+     *
+     * It's an array with document ids as keys (performance considerations)
+     *
+     * @return array
+     */
+    public function matchedDocs()
+    {
+        return array();
+    }
+
+    /**
+     * Score specified document
+     *
+     * @param integer $docId
+     * @param Zend_Search_Lucene_Interface $reader
+     * @return float
+     */
+    public function score($docId, Zend_Search_Lucene_Interface $reader)
+    {
+        return 0;
+    }
+
+    /**
+     * Return query terms
+     *
+     * @return array
+     */
+    public function getQueryTerms()
+    {
+        return array();
+    }
+
+    /**
+     * Highlight query terms
+     *
+     * @param integer &$colorIndex
+     * @param Zend_Search_Lucene_Document_Html $doc
+     */
+    public function highlightMatchesDOM(Zend_Search_Lucene_Document_Html $doc, &$colorIndex)
+    {
+        // Do nothing
+    }
+
+    /**
+     * Print a query
+     *
+     * @return string
+     */
+    public function __toString()
+    {
+        return '<EmptyQuery>';
+    }
+}
+

search/Zend/Search/Lucene/Search/Query/MultiTerm.php

@@ -15,23 +15,23 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Search_Query */
-require_once 'Zend/Search/Lucene/Search/Query.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query.php';
 
 /** Zend_Search_Lucene_Search_Weight_MultiTerm */
-require_once 'Zend/Search/Lucene/Search/Weight/MultiTerm.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/MultiTerm.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Search_Query
@@ -55,27 +55,24 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc
      *
      * @var array
      */
-
-    private $_signs = array();
+    private $_signs;
 
     /**
      * Result vector.
-     * Bitset or array of document IDs
-     * (depending from Bitset extension availability).
      *
-     * @var mixed
+     * @var array
      */
     private $_resVector = null;
 
     /**
      * Terms positions vectors.
      * Array of Arrays:
-     * term1Id => (docId => array( pos1, pos2, ... ), ...)
-     * term2Id => (docId => array( pos1, pos2, ... ), ...)
+     * term1Id => (docId => freq, ...)
+     * term2Id => (docId => freq, ...)
      *
      * @var array
      */
-    private $_termsPositions = array();
+    private $_termsFreqs = array();
 
 
     /**
@@ -101,15 +98,15 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc
     /**
      * Class constructor.  Create a new multi-term query object.
      *
+     * if $signs array is omitted then all terms are required
+     * it differs from addTerm() behavior, but should never be used
+     *
      * @param array $terms    Array of Zend_Search_Lucene_Index_Term objects
      * @param array $signs    Array of signs.  Sign is boolean|null.
      * @return void
      */
     public function __construct($terms = null, $signs = null)
     {
-        /**
-         * @todo Check contents of $terms and $signs before adding them.
-         */
         if (is_array($terms)) {
             $this->_terms = $terms;
 
@@ -119,7 +116,7 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc
                 foreach ($signs as $sign ) {
                     if ($sign !== true) {
                         $this->_signs = $signs;
-                        continue;
+                        break;
                     }
                 }
             }
@@ -139,25 +136,122 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc
      * @param  boolean|null $sign
      * @return void
      */
-    public function addTerm(Zend_Search_Lucene_Index_Term $term, $sign=null) {
-        $this->_terms[] = $term;
-
-        /**
-         * @todo This is not good.  Sometimes $this->_signs is an array, sometimes
-         * it is null, even when there are terms.  It will be changed so that
-         * it is always an array.
-         */
-        if ($this->_signs === null) {
-            if ($sign !== null) {
-                $this->_signs = array();
-                foreach ($this->_terms as $term) {
-                    $this->_signs[] = null;
+    public function addTerm(Zend_Search_Lucene_Index_Term $term, $sign = null) {
+        if ($sign !== true || $this->_signs !== null) {       // Skip, if all terms are required
+            if ($this->_signs === null) {                     // Check, If all previous terms are required
+                foreach ($this->_terms as $prevTerm) {
+                    $this->_signs[] = true;
                 }
-                $this->_signs[] = $sign;
             }
-        } else {
             $this->_signs[] = $sign;
         }
+
+        $this->_terms[] = $term;
+    }
+
+
+    /**
+     * Re-write query into primitive queries in the context of specified index
+     *
+     * @param Zend_Search_Lucene_Interface $index
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    public function rewrite(Zend_Search_Lucene_Interface $index)
+    {
+        if (count($this->_terms) == 0) {
+            return new Zend_Search_Lucene_Search_Query_Empty();
+        }
+
+        // Check, that all fields are qualified
+        $allQualified = true;
+        foreach ($this->_terms as $term) {
+            if ($term->field === null) {
+                $allQualified = false;
+                break;
+            }
+        }
+
+        if ($allQualified) {
+            return $this;
+        } else {
+            /** transform multiterm query to boolean and apply rewrite() method to subqueries. */
+            $query = new Zend_Search_Lucene_Search_Query_Boolean();
+            $query->setBoost($this->getBoost());
+
+            foreach ($this->_terms as $termId => $term) {
+                $subquery = new Zend_Search_Lucene_Search_Query_Term($term);
+
+                $query->addSubquery($subquery->rewrite($index),
+                                    ($this->_signs === null)?  true : $this->_signs[$termId]);
+            }
+
+            return $query;
+        }
+    }
+
+    /**
+     * Optimize query in the context of specified index
+     *
+     * @param Zend_Search_Lucene_Interface $index
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    public function optimize(Zend_Search_Lucene_Interface $index)
+    {
+        $terms = $this->_terms;
+        $signs = $this->_signs;
+
+        foreach ($terms as $id => $term) {
+            if (!$index->hasTerm($term)) {
+                if ($signs === null  ||  $signs[$id] === true) {
+                    // Term is required
+                    return new Zend_Search_Lucene_Search_Query_Empty();
+                } else {
+                    // Term is optional or prohibited
+                    // Remove it from terms and signs list
+                    unset($terms[$id]);
+                    unset($signs[$id]);
+                }
+            }
+        }
+
+        // Check if all presented terms are prohibited
+        $allProhibited = true;
+        if ($signs === null) {
+            $allProhibited = false;
+        } else {
+            foreach ($signs as $sign) {
+                if ($sign !== false) {
+                    $allProhibited = false;
+                    break;
+                }
+            }
+        }
+        if ($allProhibited) {
+            return new Zend_Search_Lucene_Search_Query_Empty();
+        }
+
+        /**
+         * @todo make an optimization for repeated terms
+         * (they may have different signs)
+         */
+
+        if (count($terms) == 1) {
+            // It's already checked, that it's not a prohibited term
+
+            // It's one term query with one required or optional element
+            $optimizedQuery = new Zend_Search_Lucene_Search_Query_Term(reset($terms));
+            $optimizedQuery->setBoost($this->getBoost());
+
+            return $optimizedQuery;
+        }
+
+        if (count($terms) == 0) {
+            return new Zend_Search_Lucene_Search_Query_Empty();
+        }
+
+        $optimizedQuery = new Zend_Search_Lucene_Search_Query_MultiTerm($terms, $signs);
+        $optimizedQuery->setBoost($this->getBoost());
+        return $optimizedQuery;
     }
 
 
@@ -198,12 +292,13 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc
     /**
      * Constructs an appropriate Weight implementation for this query.
      *
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      * @return Zend_Search_Lucene_Search_Weight
      */
-    protected function _createWeight($reader)
+    public function createWeight(Zend_Search_Lucene_Interface $reader)
     {
-        return new Zend_Search_Lucene_Search_Weight_MultiTerm($this, $reader);
+        $this->_weight = new Zend_Search_Lucene_Search_Weight_MultiTerm($this, $reader);
+        return $this->_weight;
     }
 
 
@@ -211,38 +306,32 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc
      * Calculate result vector for Conjunction query
      * (like '+something +another')
      *
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      */
-    private function _calculateConjunctionResult($reader)
+    private function _calculateConjunctionResult(Zend_Search_Lucene_Interface $reader)
     {
-        if (extension_loaded('bitset')) {
-            foreach( $this->_terms as $termId=>$term ) {
-                if($this->_resVector === null) {
-                    $this->_resVector = bitset_from_array($reader->termDocs($term));
-                } else {
-                    $this->_resVector = bitset_intersection(
-                                $this->_resVector,
-                                bitset_from_array($reader->termDocs($term)) );
-                }
+        $this->_resVector = null;
 
-                $this->_termsPositions[$termId] = $reader->termPositions($term);
-            }
-        } else {
-            foreach( $this->_terms as $termId=>$term ) {
-                if($this->_resVector === null) {
-                    $this->_resVector = array_flip($reader->termDocs($term));
-                } else {
-                    $termDocs = array_flip($reader->termDocs($term));
-                    foreach($this->_resVector as $key=>$value) {
-                        if (!isset( $termDocs[$key] )) {
-                            unset( $this->_resVector[$key] );
-                        }
-                    }
-                }
-
-                $this->_termsPositions[$termId] = $reader->termPositions($term);
-            }
+        if (count($this->_terms) == 0) {
+            $this->_resVector = array();
         }
+
+        foreach( $this->_terms as $termId=>$term ) {
+            if($this->_resVector === null) {
+                $this->_resVector = array_flip($reader->termDocs($term));
+            } else {
+                $this->_resVector = array_intersect_key($this->_resVector, array_flip($reader->termDocs($term)));
+            }
+
+            if (count($this->_resVector) == 0) {
+                // Empty result set, we don't need to check other terms
+                break;
+            }
+
+            $this->_termsFreqs[$termId] = $reader->termFreqs($term);
+        }
+
+        ksort($this->_resVector, SORT_NUMERIC);
     }
 
 
@@ -250,89 +339,49 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc
      * Calculate result vector for non Conjunction query
      * (like '+something -another')
      *
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      */
-    private function _calculateNonConjunctionResult($reader)
+    private function _calculateNonConjunctionResult(Zend_Search_Lucene_Interface $reader)
     {
-        if (extension_loaded('bitset')) {
-            $required   = null;
-            $neither    = bitset_empty();
-            $prohibited = bitset_empty();
+        $required   = null;
+        $optional   = array();
+        $prohibited = array();
 
-            foreach ($this->_terms as $termId => $term) {
-                $termDocs = bitset_from_array($reader->termDocs($term));
+        foreach ($this->_terms as $termId => $term) {
+            $termDocs = array_flip($reader->termDocs($term));
 
-                if ($this->_signs[$termId] === true) {
-                    // required
-                    if ($required !== null) {
-                        $required = bitset_intersection($required, $termDocs);
-                    } else {
-                        $required = $termDocs;
-                    }
-                } elseif ($this->_signs[$termId] === false) {
-                    // prohibited
-                    $prohibited = bitset_union($prohibited, $termDocs);
+            if ($this->_signs[$termId] === true) {
+                // required
+                if ($required !== null) {
+                    // array intersection
+                    $required = array_intersect_key($required, $termDocs);
                 } else {
-                    // neither required, nor prohibited
-                    $neither = bitset_union($neither, $termDocs);
+                    $required = $termDocs;
                 }
-
-                $this->_termsPositions[$termId] = $reader->termPositions($term);
+            } elseif ($this->_signs[$termId] === false) {
+                // prohibited
+                // array union
+                $prohibited += $termDocs;
+            } else {
+                // neither required, nor prohibited
+                // array union
+                $optional += $termDocs;
             }
 
-            if ($required === null) {
-                $required = $neither;
-            }
-            $this->_resVector = bitset_intersection( $required,
-                                                     bitset_invert($prohibited, $reader->count()) );
-        } else {
-            $required   = null;
-            $neither    = array();
-            $prohibited = array();
-
-            foreach ($this->_terms as $termId => $term) {
-                $termDocs = array_flip($reader->termDocs($term));
-
-                if ($this->_signs[$termId] === true) {
-                    // required
-                    if ($required !== null) {
-                        // substitute for bitset_intersection
-                        foreach ($required as $key => $value) {
-                            if (!isset( $termDocs[$key] )) {
-                                unset($required[$key]);
-                            }
-                        }
-                    } else {
-                        $required = $termDocs;
-                    }
-                } elseif ($this->_signs[$termId] === false) {
-                    // prohibited
-                    // substitute for bitset_union
-                    foreach ($termDocs as $key => $value) {
-                        $prohibited[$key] = $value;
-                    }
-                } else {
-                    // neither required, nor prohibited
-                    // substitute for bitset_union
-                    foreach ($termDocs as $key => $value) {
-                        $neither[$key] = $value;
-                    }
-                }
-
-                $this->_termsPositions[$termId] = $reader->termPositions($term);
-            }
-
-            if ($required === null) {
-                $required = $neither;
-            }
-
-            foreach ($required as $key=>$value) {
-                if (isset( $prohibited[$key] )) {
-                    unset($required[$key]);
-                }
-            }
-            $this->_resVector = $required;
+            $this->_termsFreqs[$termId] = $reader->termFreqs($term);
         }
+
+        if ($required !== null) {
+            $this->_resVector = (count($prohibited) > 0) ?
+                                           array_diff_key($required, $prohibited) :
+                                           $required;
+        } else {
+            $this->_resVector = (count($prohibited) > 0) ?
+                                           array_diff_key($optional, $prohibited) :
+                                           $optional;
+        }
+
+        ksort($this->_resVector, SORT_NUMERIC);
     }
 
 
@@ -340,10 +389,10 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc
      * Score calculator for conjunction queries (all terms are required)
      *
      * @param integer $docId
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      * @return float
      */
-    public function _conjunctionScore($docId, $reader)
+    public function _conjunctionScore($docId, Zend_Search_Lucene_Interface $reader)
     {
         if ($this->_coord === null) {
             $this->_coord = $reader->getSimilarity()->coord(count($this->_terms),
@@ -353,12 +402,16 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc
         $score = 0.0;
 
         foreach ($this->_terms as $termId=>$term) {
-            $score += $reader->getSimilarity()->tf(count($this->_termsPositions[$termId][$docId]) ) *
+            /**
+             * We don't need to check that term freq is not 0
+             * Score calculation is performed only for matched docs
+             */
+            $score += $reader->getSimilarity()->tf($this->_termsFreqs[$termId][$docId]) *
                       $this->_weights[$termId]->getValue() *
                       $reader->norm($docId, $term->field);
         }
 
-        return $score * $this->_coord;
+        return $score * $this->_coord * $this->getBoost();
     }
 
 
@@ -366,7 +419,7 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc
      * Score calculator for non conjunction queries (not all terms are required)
      *
      * @param integer $docId
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      * @return float
      */
     public function _nonConjunctionScore($docId, $reader)
@@ -390,42 +443,65 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc
         $matchedTerms = 0;
         foreach ($this->_terms as $termId=>$term) {
             // Check if term is
-            if ($this->_signs[$termId] !== false &&            // not prohibited
-                isset($this->_termsPositions[$termId][$docId]) // matched
+            if ($this->_signs[$termId] !== false &&        // not prohibited
+                isset($this->_termsFreqs[$termId][$docId]) // matched
                ) {
                 $matchedTerms++;
+
+                /**
+                 * We don't need to check that term freq is not 0
+                 * Score calculation is performed only for matched docs
+                 */
                 $score +=
-                      $reader->getSimilarity()->tf(count($this->_termsPositions[$termId][$docId]) ) *
+                      $reader->getSimilarity()->tf($this->_termsFreqs[$termId][$docId]) *
                       $this->_weights[$termId]->getValue() *
                       $reader->norm($docId, $term->field);
             }
         }
 
-        return $score * $this->_coord[$matchedTerms];
+        return $score * $this->_coord[$matchedTerms] * $this->getBoost();
+    }
+
+    /**
+     * Execute query in context of index reader
+     * It also initializes necessary internal structures
+     *
+     * @param Zend_Search_Lucene_Interface $reader
+     */
+    public function execute(Zend_Search_Lucene_Interface $reader)
+    {
+        if ($this->_signs === null) {
+            $this->_calculateConjunctionResult($reader);
+        } else {
+            $this->_calculateNonConjunctionResult($reader);
+        }
+
+        // Initialize weight if it's not done yet
+        $this->_initWeight($reader);
+    }
+
+    /**
+     * Get document ids likely matching the query
+     *
+     * It's an array with document ids as keys (performance considerations)
+     *
+     * @return array
+     */
+    public function matchedDocs()
+    {
+        return $this->_resVector;
     }
 
     /**
      * Score specified document
      *
      * @param integer $docId
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      * @return float
      */
-    public function score($docId, $reader)
+    public function score($docId, Zend_Search_Lucene_Interface $reader)
     {
-        if($this->_resVector === null) {
-            if ($this->_signs === null) {
-                $this->_calculateConjunctionResult($reader);
-            } else {
-                $this->_calculateNonConjunctionResult($reader);
-            }
-
-            $this->_initWeight($reader);
-        }
-
-        if ( (extension_loaded('bitset')) ?
-                bitset_in($this->_resVector, $docId) :
-                isset($this->_resVector[$docId])  ) {
+        if (isset($this->_resVector[$docId])) {
             if ($this->_signs === null) {
                 return $this->_conjunctionScore($docId, $reader);
             } else {
@@ -435,5 +511,87 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc
             return 0;
         }
     }
+
+    /**
+     * Return query terms
+     *
+     * @return array
+     */
+    public function getQueryTerms()
+    {
+        if ($this->_signs === null) {
+            return $this->_terms;
+        }
+
+        $terms = array();
+
+        foreach ($this->_signs as $id => $sign) {
+            if ($sign !== false) {
+                $terms[] = $this->_terms[$id];
+            }
+        }
+
+        return $terms;
+    }
+
+    /**
+     * Highlight query terms
+     *
+     * @param integer &$colorIndex
+     * @param Zend_Search_Lucene_Document_Html $doc
+     */
+    public function highlightMatchesDOM(Zend_Search_Lucene_Document_Html $doc, &$colorIndex)
+    {
+        $words = array();
+
+        if ($this->_signs === null) {
+            foreach ($this->_terms as $term) {
+                $words[] = $term->text;
+            }
+        } else {
+            foreach ($this->_signs as $id => $sign) {
+                if ($sign !== false) {
+                    $words[] = $this->_terms[$id]->text;
+                }
+            }
+        }
+
+        $doc->highlight($words, $this->_getHighlightColor($colorIndex));
+    }
+
+    /**
+     * Print a query
+     *
+     * @return string
+     */
+    public function __toString()
+    {
+        // It's used only for query visualisation, so we don't care about characters escaping
+
+        $query = '';
+
+        foreach ($this->_terms as $id => $term) {
+            if ($id != 0) {
+                $query .= ' ';
+            }
+
+            if ($this->_signs === null || $this->_signs[$id] === true) {
+                $query .= '+';
+            } else if ($this->_signs[$id] === false) {
+                $query .= '-';
+            }
+
+            if ($term->field !== null) {
+                $query .= $term->field . ':';
+            }
+            $query .= $term->text;
+        }
+
+        if ($this->getBoost() != 1) {
+            $query = '(' . $query . ')^' . $this->getBoost();
+        }
+
+        return $query;
+    }
 }
 

search/Zend/Search/Lucene/Search/Query/Phrase.php

@@ -15,7 +15,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
@@ -23,12 +23,12 @@
 /**
  * Zend_Search_Lucene_Search_Query
  */
-require_once 'Zend/Search/Lucene/Search/Query.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query.php';
 
 /**
  * Zend_Search_Lucene_Search_Weight_MultiTerm
  */
-require_once 'Zend/Search/Lucene/Search/Weight/Phrase.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/Phrase.php';
 
 
 /**
@@ -37,7 +37,7 @@ require_once 'Zend/Search/Lucene/Search/Weight/Phrase.php';
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Search_Query_Phrase extends Zend_Search_Lucene_Search_Query
@@ -73,16 +73,14 @@ class Zend_Search_Lucene_Search_Query_Phrase extends Zend_Search_Lucene_Search_Q
      *
      * The slop is zero by default, requiring exact matches.
      *
-     * @var unknown_type
+     * @var integer
      */
     private $_slop;
 
     /**
      * Result vector.
-     * Bitset or array of document IDs
-     * (depending from Bitset extension availability).
      *
-     * @var mixed
+     * @var array
      */
     private $_resVector = null;
 
@@ -183,6 +181,70 @@ class Zend_Search_Lucene_Search_Query_Phrase extends Zend_Search_Lucene_Search_Q
     }
 
 
+    /**
+     * Re-write query into primitive queries in the context of specified index
+     *
+     * @param Zend_Search_Lucene_Interface $index
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    public function rewrite(Zend_Search_Lucene_Interface $index)
+    {
+        if (count($this->_terms) == 0) {
+            return new Zend_Search_Lucene_Search_Query_Empty();
+        } else if ($this->_terms[0]->field !== null) {
+            return $this;
+        } else {
+            $query = new Zend_Search_Lucene_Search_Query_Boolean();
+            $query->setBoost($this->getBoost());
+
+            foreach ($index->getFieldNames(true) as $fieldName) {
+                $subquery = new Zend_Search_Lucene_Search_Query_Phrase();
+                $subquery->setSlop($this->getSlop());
+
+                foreach ($this->_terms as $termId => $term) {
+                    $qualifiedTerm = new Zend_Search_Lucene_Index_Term($term->text, $fieldName);
+
+                    $subquery->addTerm($qualifiedTerm, $this->_offsets[$termId]);
+                }
+
+                $query->addSubquery($subquery);
+            }
+
+            return $query;
+        }
+    }
+
+    /**
+     * Optimize query in the context of specified index
+     *
+     * @param Zend_Search_Lucene_Interface $index
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    public function optimize(Zend_Search_Lucene_Interface $index)
+    {
+        // Check, that index contains all phrase terms
+        foreach ($this->_terms as $term) {
+            if (!$index->hasTerm($term)) {
+                return new Zend_Search_Lucene_Search_Query_Empty();
+            }
+        }
+
+        if (count($this->_terms) == 1) {
+            // It's one term query
+            $optimizedQuery = new Zend_Search_Lucene_Search_Query_Term(reset($this->_terms));
+            $optimizedQuery->setBoost($this->getBoost());
+
+            return $optimizedQuery;
+        }
+
+        if (count($this->_terms) == 0) {
+            return new Zend_Search_Lucene_Search_Query_Empty();
+        }
+
+
+        return $this;
+    }
+
     /**
      * Returns query term
      *
@@ -209,50 +271,13 @@ class Zend_Search_Lucene_Search_Query_Phrase extends Zend_Search_Lucene_Search_Q
     /**
      * Constructs an appropriate Weight implementation for this query.
      *
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      * @return Zend_Search_Lucene_Search_Weight
      */
-    protected function _createWeight($reader)
+    public function createWeight(Zend_Search_Lucene_Interface $reader)
     {
-        return new Zend_Search_Lucene_Search_Weight_Phrase($this, $reader);
-    }
-
-
-    /**
-     * Calculate result vector
-     *
-     * @param Zend_Search_Lucene $reader
-     */
-    private function _calculateResult($reader)
-    {
-        if (extension_loaded('bitset')) {
-            foreach( $this->_terms as $termId=>$term ) {
-                if($this->_resVector === null) {
-                    $this->_resVector = bitset_from_array($reader->termDocs($term));
-                } else {
-                    $this->_resVector = bitset_intersection(
-                                $this->_resVector,
-                                bitset_from_array($reader->termDocs($term)) );
-                }
-
-                $this->_termsPositions[$termId] = $reader->termPositions($term);
-            }
-        } else {
-            foreach( $this->_terms as $termId=>$term ) {
-                if($this->_resVector === null) {
-                    $this->_resVector = array_flip($reader->termDocs($term));
-                } else {
-                    $termDocs = array_flip($reader->termDocs($term));
-                    foreach($this->_resVector as $key=>$value) {
-                        if (!isset( $termDocs[$key] )) {
-                            unset( $this->_resVector[$key] );
-                        }
-                    }
-                }
-
-                $this->_termsPositions[$termId] = $reader->termPositions($term);
-            }
-        }
+        $this->_weight = new Zend_Search_Lucene_Search_Weight_Phrase($this, $reader);
+        return $this->_weight;
     }
 
 
@@ -305,10 +330,10 @@ class Zend_Search_Lucene_Search_Query_Phrase extends Zend_Search_Lucene_Search_Q
      * Score calculator for sloppy phrase queries (terms sequence is fixed)
      *
      * @param integer $docId
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      * @return float
      */
-    public function _sloppyPhraseFreq($docId, Zend_Search_Lucene $reader)
+    public function _sloppyPhraseFreq($docId, Zend_Search_Lucene_Interface $reader)
     {
         $freq = 0;
 
@@ -377,50 +402,141 @@ class Zend_Search_Lucene_Search_Query_Phrase extends Zend_Search_Lucene_Search_Q
         return $freq;
     }
 
+    /**
+     * Execute query in context of index reader
+     * It also initializes necessary internal structures
+     *
+     * @param Zend_Search_Lucene_Interface $reader
+     */
+    public function execute(Zend_Search_Lucene_Interface $reader)
+    {
+        $this->_resVector = null;
+
+        if (count($this->_terms) == 0) {
+            $this->_resVector = array();
+        }
+
+        foreach( $this->_terms as $termId=>$term ) {
+            if($this->_resVector === null) {
+                $this->_resVector = array_flip($reader->termDocs($term));
+            } else {
+                $this->_resVector = array_intersect_key($this->_resVector, array_flip($reader->termDocs($term)));
+            }
+
+            if (count($this->_resVector) == 0) {
+                // Empty result set, we don't need to check other terms
+                break;
+            }
+
+            $this->_termsPositions[$termId] = $reader->termPositions($term);
+        }
+
+        ksort($this->_resVector, SORT_NUMERIC);
+
+        // Initialize weight if it's not done yet
+        $this->_initWeight($reader);
+    }
+
+    /**
+     * Get document ids likely matching the query
+     *
+     * It's an array with document ids as keys (performance considerations)
+     *
+     * @return array
+     */
+    public function matchedDocs()
+    {
+        return $this->_resVector;
+    }
 
     /**
      * Score specified document
      *
      * @param integer $docId
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      * @return float
      */
-    public function score($docId, $reader)
+    public function score($docId, Zend_Search_Lucene_Interface $reader)
     {
-        // optimize zero-term case
-        if (count($this->_terms) == 0) {
-            return 0;
-        }
-
-        if($this->_resVector === null) {
-            $this->_calculateResult($reader);
-            $this->_initWeight($reader);
-        }
-
-        if ( (extension_loaded('bitset')) ?
-                bitset_in($this->_resVector, $docId) :
-                isset($this->_resVector[$docId])  ) {
+        if (isset($this->_resVector[$docId])) {
             if ($this->_slop == 0) {
                 $freq = $this->_exactPhraseFreq($docId);
             } else {
                 $freq = $this->_sloppyPhraseFreq($docId, $reader);
             }
 
-/*
-            return $reader->getSimilarity()->tf($freq) *
-                   $this->_weight->getValue() *
-                   $reader->norm($docId, reset($this->_terms)->field);
-*/
             if ($freq != 0) {
                 $tf = $reader->getSimilarity()->tf($freq);
                 $weight = $this->_weight->getValue();
                 $norm = $reader->norm($docId, reset($this->_terms)->field);
 
-                return $tf*$weight*$norm;
+                return $tf * $weight * $norm * $this->getBoost();
             }
+
+            // Included in result, but culculated freq is zero
+            return 0;
         } else {
             return 0;
         }
     }
+
+    /**
+     * Return query terms
+     *
+     * @return array
+     */
+    public function getQueryTerms()
+    {
+        return $this->_terms;
+    }
+
+    /**
+     * Highlight query terms
+     *
+     * @param integer &$colorIndex
+     * @param Zend_Search_Lucene_Document_Html $doc
+     */
+    public function highlightMatchesDOM(Zend_Search_Lucene_Document_Html $doc, &$colorIndex)
+    {
+        $words = array();
+        foreach ($this->_terms as $term) {
+            $words[] = $term->text;
+        }
+
+        $doc->highlight($words, $this->_getHighlightColor($colorIndex));
+    }
+
+    /**
+     * Print a query
+     *
+     * @return string
+     */
+    public function __toString()
+    {
+        // It's used only for query visualisation, so we don't care about characters escaping
+
+        $query = '';
+
+        if (isset($this->_terms[0]) && $this->_terms[0]->field !== null) {
+            $query .= $this->_terms[0]->field . ':';
+        }
+
+        $query .= '"';
+
+        foreach ($this->_terms as $id => $term) {
+            if ($id != 0) {
+                $query .= ' ';
+            }
+            $query .= $term->text;
+        }
+
+        $query .= '"';
+
+        if ($this->_slop != 0) {
+            $query .= '~' . $this->_slop;
+        }
+
+        return $query;
+    }
 }
 

search/Zend/Search/Lucene/Search/Query/Term.php

@@ -15,23 +15,23 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Search_Query */
-require_once 'Zend/Search/Lucene/Search/Query.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query.php';
 
 /** Zend_Search_Lucene_Search_Weight_Term */
-require_once 'Zend/Search/Lucene/Search/Weight/Term.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/Term.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Search_Query_Term extends Zend_Search_Lucene_Search_Query
@@ -43,31 +43,20 @@ class Zend_Search_Lucene_Search_Query_Term extends Zend_Search_Lucene_Search_Que
      */
     private $_term;
 
-    /**
-     * Term sign.
-     * If true then term is required
-     * If false then term is prohibited.
-     *
-     * @var bool
-     */
-    private $_sign;
-
     /**
      * Documents vector.
-     * Bitset or array of document IDs
-     * (depending from Bitset extension availability).
      *
-     * @var mixed
+     * @var array
      */
     private $_docVector = null;
 
     /**
-     * Term positions vector.
-     * Array: docId => array( pos1, pos2, ... )
+     * Term freqs vector.
+     * array(docId => freq, ...)
      *
      * @var array
      */
-    private $_termPositions;
+    private $_termFreqs;
 
 
     /**
@@ -76,53 +65,160 @@ class Zend_Search_Lucene_Search_Query_Term extends Zend_Search_Lucene_Search_Que
      * @param Zend_Search_Lucene_Index_Term $term
      * @param boolean $sign
      */
-    public function __construct( $term, $sign = true )
+    public function __construct($term)
     {
         $this->_term = $term;
-        $this->_sign = $sign;
+    }
+
+    /**
+     * Re-write query into primitive queries in the context of specified index
+     *
+     * @param Zend_Search_Lucene_Interface $index
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    public function rewrite(Zend_Search_Lucene_Interface $index)
+    {
+        if ($this->_term->field != null) {
+            return $this;
+        } else {
+            $query = new Zend_Search_Lucene_Search_Query_MultiTerm();
+            $query->setBoost($this->getBoost());
+
+            foreach ($index->getFieldNames(true) as $fieldName) {
+                $term = new Zend_Search_Lucene_Index_Term($this->_term->text, $fieldName);
+
+                $query->addTerm($term);
+            }
+
+            return $query->rewrite($index);
+        }
+    }
+
+    /**
+     * Optimize query in the context of specified index
+     *
+     * @param Zend_Search_Lucene_Interface $index
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    public function optimize(Zend_Search_Lucene_Interface $index)
+    {
+        // Check, that index contains specified term
+        if (!$index->hasTerm($this->_term)) {
+            return new Zend_Search_Lucene_Search_Query_Empty();
+        }
+
+        return $this;
     }
 
 
     /**
      * Constructs an appropriate Weight implementation for this query.
      *
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      * @return Zend_Search_Lucene_Search_Weight
      */
-    protected function _createWeight($reader)
+    public function createWeight(Zend_Search_Lucene_Interface $reader)
     {
-        return new Zend_Search_Lucene_Search_Weight_Term($this->_term, $this, $reader);
+        $this->_weight = new Zend_Search_Lucene_Search_Weight_Term($this->_term, $this, $reader);
+        return $this->_weight;
+    }
+
+    /**
+     * Execute query in context of index reader
+     * It also initializes necessary internal structures
+     *
+     * @param Zend_Search_Lucene_Interface $reader
+     */
+    public function execute(Zend_Search_Lucene_Interface $reader)
+    {
+        $this->_docVector = array_flip($reader->termDocs($this->_term));
+        $this->_termFreqs = $reader->termFreqs($this->_term);
+
+        // Initialize weight if it's not done yet
+        $this->_initWeight($reader);
+    }
+
+    /**
+     * Get document ids likely matching the query
+     *
+     * It's an array with document ids as keys (performance considerations)
+     *
+     * @return array
+     */
+    public function matchedDocs()
+    {
+        return $this->_docVector;
     }
 
     /**
      * Score specified document
      *
      * @param integer $docId
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      * @return float
      */
-    public function score( $docId, $reader )
+    public function score($docId, Zend_Search_Lucene_Interface $reader)
     {
-        if($this->_docVector===null) {
-            if (extension_loaded('bitset')) {
-                $this->_docVector = bitset_from_array( $reader->termDocs($this->_term) );
-            } else {
-                $this->_docVector = array_flip($reader->termDocs($this->_term));
-            }
-
-            $this->_termPositions = $reader->termPositions($this->_term);
-            $this->_initWeight($reader);
-        }
-
-        $match = extension_loaded('bitset') ?  bitset_in($this->_docVector, $docId) :
-                                               isset($this->_docVector[$docId]);
-        if ($this->_sign && $match) {
-            return $reader->getSimilarity()->tf(count($this->_termPositions[$docId]) ) *
+        if (isset($this->_docVector[$docId])) {
+            return $reader->getSimilarity()->tf($this->_termFreqs[$docId]) *
                    $this->_weight->getValue() *
-                   $reader->norm($docId, $this->_term->field);
+                   $reader->norm($docId, $this->_term->field) *
+                   $this->getBoost();
         } else {
             return 0;
         }
     }
+
+    /**
+     * Return query terms
+     *
+     * @return array
+     */
+    public function getQueryTerms()
+    {
+        return array($this->_term);
+    }
+
+    /**
+     * Return query term
+     *
+     * @return Zend_Search_Lucene_Index_Term
+     */
+    public function getTerm()
+    {
+        return $this->_term;
+    }
+
+    /**
+     * Returns query term
+     *
+     * @return array
+     */
+    public function getTerms()
+    {
+        return $this->_terms;
+    }
+
+    /**
+     * Highlight query terms
+     *
+     * @param integer &$colorIndex
+     * @param Zend_Search_Lucene_Document_Html $doc
+     */
+    public function highlightMatchesDOM(Zend_Search_Lucene_Document_Html $doc, &$colorIndex)
+    {
+        $doc->highlight($this->_term->text, $this->_getHighlightColor($colorIndex));
+    }
+
+    /**
+     * Print a query
+     *
+     * @return string
+     */
+    public function __toString()
+    {
+        // It's used only for query visualisation, so we don't care about characters escaping
+        return (($this->_term->field === null)? '':$this->_term->field . ':') . $this->_term->text;
+    }
 }
 

search/Zend/Search/Lucene/Search/QueryEntry.php

@@ -0,0 +1,87 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Index_Term */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php';
+
+/** Zend_Search_Lucene_Exception */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+/** Zend_Search_Lucene_Search_QueryEntry_Term */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry/Term.php';
+
+/** Zend_Search_Lucene_Search_QueryEntry_Phrase */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry/Phrase.php';
+
+/** Zend_Search_Lucene_Search_QueryEntry_Subquery */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry/Subquery.php';
+
+
+/** Zend_Search_Lucene_Search_QueryParserException */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+abstract class Zend_Search_Lucene_Search_QueryEntry
+{
+    /**
+     * Query entry boost factor
+     *
+     * @var float
+     */
+    protected $_boost = 1.0;
+
+
+    /**
+     * Process modifier ('~')
+     *
+     * @param mixed $parameter
+     */
+    abstract public function processFuzzyProximityModifier($parameter = null);
+
+
+    /**
+     * Transform entry to a subquery
+     *
+     * @param string $encoding
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    abstract public function getQuery($encoding);
+
+    /**
+     * Boost query entry
+     *
+     * @param float $boostFactor
+     */
+    public function boost($boostFactor)
+    {
+        $this->_boost *= $boostFactor;
+    }
+
+
+}

search/Zend/Search/Lucene/Search/QueryEntry/Phrase.php

@@ -0,0 +1,147 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Index_Term */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php';
+
+/** Zend_Search_Lucene_Exception */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+/** Zend_Search_Lucene_Search_QueryEntry */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry.php';
+
+/** Zend_Search_Lucene_Search_QueryParserException */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php';
+
+/** Zend_Search_Lucene_Analysis_Analyzer */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php';
+
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Search_QueryEntry_Phrase extends Zend_Search_Lucene_Search_QueryEntry
+{
+    /**
+     * Phrase value
+     *
+     * @var string
+     */
+    private $_phrase;
+
+    /**
+     * Field
+     *
+     * @var string|null
+     */
+    private $_field;
+
+
+    /**
+     * Proximity phrase query
+     *
+     * @var boolean
+     */
+    private $_proximityQuery = false;
+
+    /**
+     * Words distance, used for proximiti queries
+     *
+     * @var integer
+     */
+    private $_wordsDistance = 0;
+
+
+    /**
+     * Object constractor
+     *
+     * @param string $phrase
+     * @param string $field
+     */
+    public function __construct($phrase, $field)
+    {
+        $this->_phrase = $phrase;
+        $this->_field  = $field;
+    }
+
+    /**
+     * Process modifier ('~')
+     *
+     * @param mixed $parameter
+     */
+    public function processFuzzyProximityModifier($parameter = null)
+    {
+        $this->_proximityQuery = true;
+
+        if ($parameter !== null) {
+            $this->_wordsDistance = $parameter;
+        }
+    }
+
+    /**
+     * Transform entry to a subquery
+     *
+     * @param string $encoding
+     * @return Zend_Search_Lucene_Search_Query
+     * @throws Zend_Search_Lucene_Search_QueryParserException
+     */
+    public function getQuery($encoding)
+    {
+        if (strpos($this->_phrase, '?') !== false || strpos($this->_phrase, '*') !== false) {
+            throw new Zend_Search_Lucene_Search_QueryParserException('Wildcards are only allowed in a single terms.');
+        }
+
+        $tokens = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($this->_phrase, $encoding);
+
+        if (count($tokens) == 0) {
+            return new Zend_Search_Lucene_Search_Query_Empty();
+        }
+
+        if (count($tokens) == 1) {
+            $term  = new Zend_Search_Lucene_Index_Term($tokens[0]->getTermText(), $this->_field);
+            $query = new Zend_Search_Lucene_Search_Query_Term($term);
+            $query->setBoost($this->_boost);
+
+            return $query;
+        }
+
+        //It's not empty or one term query
+        $query = new Zend_Search_Lucene_Search_Query_Phrase();
+        foreach ($tokens as $token) {
+            $term = new Zend_Search_Lucene_Index_Term($token->getTermText(), $this->_field);
+            $query->addTerm($term);
+        }
+
+        if ($this->_proximityQuery) {
+            $query->setSlop($this->_wordsDistance);
+        }
+
+        $query->setBoost($this->_boost);
+
+        return $query;
+    }
+}

search/Zend/Search/Lucene/Search/QueryEntry/Subquery.php

@@ -0,0 +1,86 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Index_Term */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php';
+
+/** Zend_Search_Lucene_Exception */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+/** Zend_Search_Lucene_Search_QueryEntry */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry.php';
+
+/** Zend_Search_Lucene_Search_QueryParserException */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Search_QueryEntry_Subquery extends Zend_Search_Lucene_Search_QueryEntry
+{
+    /**
+     * Query
+     *
+     * @var Zend_Search_Lucene_Search_Query
+     */
+    private $_query;
+
+    /**
+     * Object constractor
+     *
+     * @param Zend_Search_Lucene_Search_Query $query
+     */
+    public function __construct(Zend_Search_Lucene_Search_Query $query)
+    {
+        $this->_query = $query;
+    }
+
+    /**
+     * Process modifier ('~')
+     *
+     * @param mixed $parameter
+     * @throws Zend_Search_Lucene_Search_QueryParserException
+     */
+    public function processFuzzyProximityModifier($parameter = null)
+    {
+        throw new Zend_Search_Lucene_Search_QueryParserException('\'~\' sign must follow term or phrase');
+    }
+
+
+    /**
+     * Transform entry to a subquery
+     *
+     * @param string $encoding
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    public function getQuery($encoding)
+    {
+        $this->_query->setBoost($this->_boost);
+
+        return $this->_query;
+    }
+}

search/Zend/Search/Lucene/Search/QueryEntry/Term.php

@@ -0,0 +1,154 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Index_Term */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php';
+
+/** Zend_Search_Lucene_Exception */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+/** Zend_Search_Lucene_Search_QueryEntry */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry.php';
+
+/** Zend_Search_Lucene_Search_QueryParserException */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php';
+
+/** Zend_Search_Lucene_Analysis_Analyzer */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php';
+
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Search_QueryEntry_Term extends Zend_Search_Lucene_Search_QueryEntry
+{
+    /**
+     * Term value
+     *
+     * @var string
+     */
+    private $_term;
+
+    /**
+     * Field
+     *
+     * @var string|null
+     */
+    private $_field;
+
+
+    /**
+     * Fuzzy search query
+     *
+     * @var boolean
+     */
+    private $_fuzzyQuery = false;
+
+    /**
+     * Similarity
+     *
+     * @var float
+     */
+    private $_similarity = 1.;
+
+
+    /**
+     * Object constractor
+     *
+     * @param string $term
+     * @param string $field
+     */
+    public function __construct($term, $field)
+    {
+        $this->_term  = $term;
+        $this->_field = $field;
+    }
+
+    /**
+     * Process modifier ('~')
+     *
+     * @param mixed $parameter
+     */
+    public function processFuzzyProximityModifier($parameter = null)
+    {
+        $this->_fuzzyQuery = true;
+
+        if ($parameter !== null) {
+            $this->_similarity = $parameter;
+        } else {
+            $this->_similarity = 0.5;
+        }
+    }
+
+    /**
+     * Transform entry to a subquery
+     *
+     * @param string $encoding
+     * @return Zend_Search_Lucene_Search_Query
+     * @throws Zend_Search_Lucene_Search_QueryParserException
+     */
+    public function getQuery($encoding)
+    {
+        if ($this->_fuzzyQuery) {
+            throw new Zend_Search_Lucene_Search_QueryParserException('Fuzzy search is not supported yet.');
+        }
+
+        if (strpos($this->_term, '?') !== false || strpos($this->_term, '*') !== false) {
+            throw new Zend_Search_Lucene_Search_QueryParserException('Wildcard queries are not supported yet.');
+        }
+
+        $tokens = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($this->_term, $encoding);
+
+        if (count($tokens) == 0) {
+            return new Zend_Search_Lucene_Search_Query_Empty();
+        }
+
+        if (count($tokens) == 1) {
+            $term  = new Zend_Search_Lucene_Index_Term($tokens[0]->getTermText(), $this->_field);
+            $query = new Zend_Search_Lucene_Search_Query_Term($term);
+            $query->setBoost($this->_boost);
+
+            return $query;
+        }
+
+        //It's not empty or one term query
+        $query = new Zend_Search_Lucene_Search_Query_MultiTerm();
+
+        /**
+         * @todo Process $token->getPositionIncrement() to support stemming, synonyms and other
+         * analizer design features
+         */
+        foreach ($tokens as $token) {
+            $term = new Zend_Search_Lucene_Index_Term($token->getTermText(), $this->_field);
+            $query->addTerm($term, true); // all subterms are required
+        }
+
+        $query->setBoost($this->_boost);
+
+        return $query;
+    }
+}

search/Zend/Search/Lucene/Search/QueryHit.php

@@ -15,7 +15,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
@@ -24,14 +24,14 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Search_QueryHit
 {
     /**
      * Object handle of the index
-     * @var Zend_Search_Lucene
+     * @var Zend_Search_Lucene_Interface
      */
     protected $_index = null;
 
@@ -55,15 +55,15 @@ class Zend_Search_Lucene_Search_QueryHit
 
 
     /**
-     * Constructor - pass object handle of Zend_Search_Lucene index that produced
+     * Constructor - pass object handle of Zend_Search_Lucene_Interface index that produced
      * the hit so the document can be retrieved easily from the hit.
      *
-     * @param Zend_Search_Lucene $index
+     * @param Zend_Search_Lucene_Interface $index
      */
 
-    public function __construct(Zend_Search_Lucene $index)
+    public function __construct(Zend_Search_Lucene_Interface $index)
     {
-        $this->_index = $index;
+        $this->_index = new Zend_Search_Lucene_Proxy($index);
     }
 
 
@@ -98,7 +98,7 @@ class Zend_Search_Lucene_Search_QueryHit
     /**
      * Return the index object for this hit
      *
-     * @return Zend_Search_Lucene
+     * @return Zend_Search_Lucene_Interface
      */
     public function getIndex()
     {

search/Zend/Search/Lucene/Search/QueryLexer.php

@@ -0,0 +1,508 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_FSM */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/FSM.php';
+
+/** Zend_Search_Lucene_Search_QueryParser */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryToken.php';
+
+/** Zend_Search_Lucene_Exception */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+/** Zend_Search_Lucene_Search_QueryParserException */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Search_QueryLexer extends Zend_Search_Lucene_FSM
+{
+    /** State Machine states */
+    const ST_WHITE_SPACE     = 0;
+    const ST_SYNT_LEXEME     = 1;
+    const ST_LEXEME          = 2;
+    const ST_QUOTED_LEXEME   = 3;
+    const ST_ESCAPED_CHAR    = 4;
+    const ST_ESCAPED_QCHAR   = 5;
+    const ST_LEXEME_MODIFIER = 6;
+    const ST_NUMBER          = 7;
+    const ST_MANTISSA        = 8;
+    const ST_ERROR           = 9;
+
+    /** Input symbols */
+    const IN_WHITE_SPACE     = 0;
+    const IN_SYNT_CHAR       = 1;
+    const IN_LEXEME_MODIFIER = 2;
+    const IN_ESCAPE_CHAR     = 3;
+    const IN_QUOTE           = 4;
+    const IN_DECIMAL_POINT   = 5;
+    const IN_ASCII_DIGIT     = 6;
+    const IN_CHAR            = 7;
+    const IN_MUTABLE_CHAR    = 8;
+
+    const QUERY_WHITE_SPACE_CHARS      = " \n\r\t";
+    const QUERY_SYNT_CHARS             = ':()[]{}!|&';
+    const QUERY_MUTABLE_CHARS          = '+-';
+    const QUERY_DOUBLECHARLEXEME_CHARS = '|&';
+    const QUERY_LEXEMEMODIFIER_CHARS   = '~^';
+    const QUERY_ASCIIDIGITS_CHARS      = '0123456789';
+
+    /**
+     * List of recognized lexemes
+     *
+     * @var array
+     */
+    private $_lexemes;
+
+    /**
+     * Query string (array of single- or non single-byte characters)
+     *
+     * @var array
+     */
+    private $_queryString;
+
+    /**
+     * Current position within a query string
+     * Used to create appropriate error messages
+     *
+     * @var integer
+     */
+    private $_queryStringPosition;
+
+    /**
+     * Recognized part of current lexeme
+     *
+     * @var string
+     */
+    private $_currentLexeme;
+
+    public function __construct()
+    {
+        parent::__construct( array(self::ST_WHITE_SPACE,
+                                   self::ST_SYNT_LEXEME,
+                                   self::ST_LEXEME,
+                                   self::ST_QUOTED_LEXEME,
+                                   self::ST_ESCAPED_CHAR,
+                                   self::ST_ESCAPED_QCHAR,
+                                   self::ST_LEXEME_MODIFIER,
+                                   self::ST_NUMBER,
+                                   self::ST_MANTISSA,
+                                   self::ST_ERROR),
+                             array(self::IN_WHITE_SPACE,
+                                   self::IN_SYNT_CHAR,
+                                   self::IN_MUTABLE_CHAR,
+                                   self::IN_LEXEME_MODIFIER,
+                                   self::IN_ESCAPE_CHAR,
+                                   self::IN_QUOTE,
+                                   self::IN_DECIMAL_POINT,
+                                   self::IN_ASCII_DIGIT,
+                                   self::IN_CHAR));
+
+
+        $lexemeModifierErrorAction    = new Zend_Search_Lucene_FSMAction($this, 'lexModifierErrException');
+        $quoteWithinLexemeErrorAction = new Zend_Search_Lucene_FSMAction($this, 'quoteWithinLexemeErrException');
+        $wrongNumberErrorAction       = new Zend_Search_Lucene_FSMAction($this, 'wrongNumberErrException');
+
+
+
+        $this->addRules(array( array(self::ST_WHITE_SPACE,   self::IN_WHITE_SPACE,     self::ST_WHITE_SPACE),
+                               array(self::ST_WHITE_SPACE,   self::IN_SYNT_CHAR,       self::ST_SYNT_LEXEME),
+                               array(self::ST_WHITE_SPACE,   self::IN_MUTABLE_CHAR,    self::ST_SYNT_LEXEME),
+                               array(self::ST_WHITE_SPACE,   self::IN_LEXEME_MODIFIER, self::ST_LEXEME_MODIFIER),
+                               array(self::ST_WHITE_SPACE,   self::IN_ESCAPE_CHAR,     self::ST_ESCAPED_CHAR),
+                               array(self::ST_WHITE_SPACE,   self::IN_QUOTE,           self::ST_QUOTED_LEXEME),
+                               array(self::ST_WHITE_SPACE,   self::IN_DECIMAL_POINT,   self::ST_LEXEME),
+                               array(self::ST_WHITE_SPACE,   self::IN_ASCII_DIGIT,     self::ST_LEXEME),
+                               array(self::ST_WHITE_SPACE,   self::IN_CHAR,            self::ST_LEXEME)
+                             ));
+        $this->addRules(array( array(self::ST_SYNT_LEXEME,   self::IN_WHITE_SPACE,     self::ST_WHITE_SPACE),
+                               array(self::ST_SYNT_LEXEME,   self::IN_SYNT_CHAR,       self::ST_SYNT_LEXEME),
+                               array(self::ST_SYNT_LEXEME,   self::IN_MUTABLE_CHAR,    self::ST_SYNT_LEXEME),
+                               array(self::ST_SYNT_LEXEME,   self::IN_LEXEME_MODIFIER, self::ST_LEXEME_MODIFIER),
+                               array(self::ST_SYNT_LEXEME,   self::IN_ESCAPE_CHAR,     self::ST_ESCAPED_CHAR),
+                               array(self::ST_SYNT_LEXEME,   self::IN_QUOTE,           self::ST_QUOTED_LEXEME),
+                               array(self::ST_SYNT_LEXEME,   self::IN_DECIMAL_POINT,   self::ST_LEXEME),
+                               array(self::ST_SYNT_LEXEME,   self::IN_ASCII_DIGIT,     self::ST_LEXEME),
+                               array(self::ST_SYNT_LEXEME,   self::IN_CHAR,            self::ST_LEXEME)
+                             ));
+        $this->addRules(array( array(self::ST_LEXEME,        self::IN_WHITE_SPACE,     self::ST_WHITE_SPACE),
+                               array(self::ST_LEXEME,        self::IN_SYNT_CHAR,       self::ST_SYNT_LEXEME),
+                               array(self::ST_LEXEME,        self::IN_MUTABLE_CHAR,    self::ST_LEXEME),
+                               array(self::ST_LEXEME,        self::IN_LEXEME_MODIFIER, self::ST_LEXEME_MODIFIER),
+                               array(self::ST_LEXEME,        self::IN_ESCAPE_CHAR,     self::ST_ESCAPED_CHAR),
+
+                               // IN_QUOTE     not allowed
+                               array(self::ST_LEXEME,        self::IN_QUOTE,           self::ST_ERROR, $quoteWithinLexemeErrorAction),
+
+                               array(self::ST_LEXEME,        self::IN_DECIMAL_POINT,   self::ST_LEXEME),
+                               array(self::ST_LEXEME,        self::IN_ASCII_DIGIT,     self::ST_LEXEME),
+                               array(self::ST_LEXEME,        self::IN_CHAR,            self::ST_LEXEME)
+                             ));
+        $this->addRules(array( array(self::ST_QUOTED_LEXEME, self::IN_WHITE_SPACE,     self::ST_QUOTED_LEXEME),
+                               array(self::ST_QUOTED_LEXEME, self::IN_SYNT_CHAR,       self::ST_QUOTED_LEXEME),
+                               array(self::ST_QUOTED_LEXEME, self::IN_MUTABLE_CHAR,    self::ST_QUOTED_LEXEME),
+                               array(self::ST_QUOTED_LEXEME, self::IN_LEXEME_MODIFIER, self::ST_QUOTED_LEXEME),
+                               array(self::ST_QUOTED_LEXEME, self::IN_ESCAPE_CHAR,     self::ST_ESCAPED_QCHAR),
+                               array(self::ST_QUOTED_LEXEME, self::IN_QUOTE,           self::ST_WHITE_SPACE),
+                               array(self::ST_QUOTED_LEXEME, self::IN_DECIMAL_POINT,   self::ST_QUOTED_LEXEME),
+                               array(self::ST_QUOTED_LEXEME, self::IN_ASCII_DIGIT,     self::ST_QUOTED_LEXEME),
+                               array(self::ST_QUOTED_LEXEME, self::IN_CHAR,            self::ST_QUOTED_LEXEME)
+                             ));
+        $this->addRules(array( array(self::ST_ESCAPED_CHAR,  self::IN_WHITE_SPACE,     self::ST_LEXEME),
+                               array(self::ST_ESCAPED_CHAR,  self::IN_SYNT_CHAR,       self::ST_LEXEME),
+                               array(self::ST_ESCAPED_CHAR,  self::IN_MUTABLE_CHAR,    self::ST_LEXEME),
+                               array(self::ST_ESCAPED_CHAR,  self::IN_LEXEME_MODIFIER, self::ST_LEXEME),
+                               array(self::ST_ESCAPED_CHAR,  self::IN_ESCAPE_CHAR,     self::ST_LEXEME),
+                               array(self::ST_ESCAPED_CHAR,  self::IN_QUOTE,           self::ST_LEXEME),
+                               array(self::ST_ESCAPED_CHAR,  self::IN_DECIMAL_POINT,   self::ST_LEXEME),
+                               array(self::ST_ESCAPED_CHAR,  self::IN_ASCII_DIGIT,     self::ST_LEXEME),
+                               array(self::ST_ESCAPED_CHAR,  self::IN_CHAR,            self::ST_LEXEME)
+                             ));
+        $this->addRules(array( array(self::ST_ESCAPED_QCHAR, self::IN_WHITE_SPACE,     self::ST_QUOTED_LEXEME),
+                               array(self::ST_ESCAPED_QCHAR, self::IN_SYNT_CHAR,       self::ST_QUOTED_LEXEME),
+                               array(self::ST_ESCAPED_QCHAR, self::IN_MUTABLE_CHAR,    self::ST_QUOTED_LEXEME),
+                               array(self::ST_ESCAPED_QCHAR, self::IN_LEXEME_MODIFIER, self::ST_QUOTED_LEXEME),
+                               array(self::ST_ESCAPED_QCHAR, self::IN_ESCAPE_CHAR,     self::ST_QUOTED_LEXEME),
+                               array(self::ST_ESCAPED_QCHAR, self::IN_QUOTE,           self::ST_QUOTED_LEXEME),
+                               array(self::ST_ESCAPED_QCHAR, self::IN_DECIMAL_POINT,   self::ST_QUOTED_LEXEME),
+                               array(self::ST_ESCAPED_QCHAR, self::IN_ASCII_DIGIT,     self::ST_QUOTED_LEXEME),
+                               array(self::ST_ESCAPED_QCHAR, self::IN_CHAR,            self::ST_QUOTED_LEXEME)
+                             ));
+        $this->addRules(array( array(self::ST_LEXEME_MODIFIER, self::IN_WHITE_SPACE,     self::ST_WHITE_SPACE),
+                               array(self::ST_LEXEME_MODIFIER, self::IN_SYNT_CHAR,       self::ST_SYNT_LEXEME),
+                               array(self::ST_LEXEME_MODIFIER, self::IN_MUTABLE_CHAR,    self::ST_SYNT_LEXEME),
+                               array(self::ST_LEXEME_MODIFIER, self::IN_LEXEME_MODIFIER, self::ST_LEXEME_MODIFIER),
+
+                               // IN_ESCAPE_CHAR       not allowed
+                               array(self::ST_LEXEME_MODIFIER, self::IN_ESCAPE_CHAR,     self::ST_ERROR, $lexemeModifierErrorAction),
+
+                               // IN_QUOTE             not allowed
+                               array(self::ST_LEXEME_MODIFIER, self::IN_QUOTE,           self::ST_ERROR, $lexemeModifierErrorAction),
+
+
+                               array(self::ST_LEXEME_MODIFIER, self::IN_DECIMAL_POINT,   self::ST_MANTISSA),
+                               array(self::ST_LEXEME_MODIFIER, self::IN_ASCII_DIGIT,     self::ST_NUMBER),
+
+                               // IN_CHAR              not allowed
+                               array(self::ST_LEXEME_MODIFIER, self::IN_CHAR,            self::ST_ERROR, $lexemeModifierErrorAction),
+                             ));
+        $this->addRules(array( array(self::ST_NUMBER, self::IN_WHITE_SPACE,     self::ST_WHITE_SPACE),
+                               array(self::ST_NUMBER, self::IN_SYNT_CHAR,       self::ST_SYNT_LEXEME),
+                               array(self::ST_NUMBER, self::IN_MUTABLE_CHAR,    self::ST_SYNT_LEXEME),
+                               array(self::ST_NUMBER, self::IN_LEXEME_MODIFIER, self::ST_LEXEME_MODIFIER),
+
+                               // IN_ESCAPE_CHAR       not allowed
+                               array(self::ST_NUMBER, self::IN_ESCAPE_CHAR,     self::ST_ERROR, $wrongNumberErrorAction),
+
+                               // IN_QUOTE             not allowed
+                               array(self::ST_NUMBER, self::IN_QUOTE,           self::ST_ERROR, $wrongNumberErrorAction),
+
+                               array(self::ST_NUMBER, self::IN_DECIMAL_POINT,   self::ST_MANTISSA),
+                               array(self::ST_NUMBER, self::IN_ASCII_DIGIT,     self::ST_NUMBER),
+
+                               // IN_CHAR              not allowed
+                               array(self::ST_NUMBER, self::IN_CHAR,            self::ST_ERROR, $wrongNumberErrorAction),
+                             ));
+        $this->addRules(array( array(self::ST_MANTISSA, self::IN_WHITE_SPACE,     self::ST_WHITE_SPACE),
+                               array(self::ST_MANTISSA, self::IN_SYNT_CHAR,       self::ST_SYNT_LEXEME),
+                               array(self::ST_MANTISSA, self::IN_MUTABLE_CHAR,    self::ST_SYNT_LEXEME),
+                               array(self::ST_MANTISSA, self::IN_LEXEME_MODIFIER, self::ST_LEXEME_MODIFIER),
+
+                               // IN_ESCAPE_CHAR       not allowed
+                               array(self::ST_MANTISSA, self::IN_ESCAPE_CHAR,     self::ST_ERROR, $wrongNumberErrorAction),
+
+                               // IN_QUOTE             not allowed
+                               array(self::ST_MANTISSA, self::IN_QUOTE,           self::ST_ERROR, $wrongNumberErrorAction),
+
+                               // IN_DECIMAL_POINT     not allowed
+                               array(self::ST_MANTISSA, self::IN_DECIMAL_POINT,   self::ST_ERROR, $wrongNumberErrorAction),
+
+                               array(self::ST_MANTISSA, self::IN_ASCII_DIGIT,     self::ST_MANTISSA),
+
+                               // IN_CHAR              not allowed
+                               array(self::ST_MANTISSA, self::IN_CHAR,            self::ST_ERROR, $wrongNumberErrorAction),
+                             ));
+
+
+        /** Actions */
+        $syntaxLexemeAction    = new Zend_Search_Lucene_FSMAction($this, 'addQuerySyntaxLexeme');
+        $lexemeModifierAction  = new Zend_Search_Lucene_FSMAction($this, 'addLexemeModifier');
+        $addLexemeAction       = new Zend_Search_Lucene_FSMAction($this, 'addLexeme');
+        $addQuotedLexemeAction = new Zend_Search_Lucene_FSMAction($this, 'addQuotedLexeme');
+        $addNumberLexemeAction = new Zend_Search_Lucene_FSMAction($this, 'addNumberLexeme');
+        $addLexemeCharAction   = new Zend_Search_Lucene_FSMAction($this, 'addLexemeChar');
+
+
+        /** Syntax lexeme */
+        $this->addEntryAction(self::ST_SYNT_LEXEME,  $syntaxLexemeAction);
+        // Two lexemes in succession
+        $this->addTransitionAction(self::ST_SYNT_LEXEME, self::ST_SYNT_LEXEME, $syntaxLexemeAction);
+
+
+        /** Lexeme */
+        $this->addEntryAction(self::ST_LEXEME,                       $addLexemeCharAction);
+        $this->addTransitionAction(self::ST_LEXEME, self::ST_LEXEME, $addLexemeCharAction);
+        // ST_ESCAPED_CHAR => ST_LEXEME transition is covered by ST_LEXEME entry action
+
+        $this->addTransitionAction(self::ST_LEXEME, self::ST_WHITE_SPACE,     $addLexemeAction);
+        $this->addTransitionAction(self::ST_LEXEME, self::ST_SYNT_LEXEME,     $addLexemeAction);
+        $this->addTransitionAction(self::ST_LEXEME, self::ST_QUOTED_LEXEME,   $addLexemeAction);
+        $this->addTransitionAction(self::ST_LEXEME, self::ST_LEXEME_MODIFIER, $addLexemeAction);
+        $this->addTransitionAction(self::ST_LEXEME, self::ST_NUMBER,          $addLexemeAction);
+        $this->addTransitionAction(self::ST_LEXEME, self::ST_MANTISSA,        $addLexemeAction);
+
+
+        /** Quoted lexeme */
+        // We don't need entry action (skeep quote)
+        $this->addTransitionAction(self::ST_QUOTED_LEXEME, self::ST_QUOTED_LEXEME, $addLexemeCharAction);
+        $this->addTransitionAction(self::ST_ESCAPED_QCHAR, self::ST_QUOTED_LEXEME, $addLexemeCharAction);
+        // Closing quote changes state to the ST_WHITE_SPACE   other states are not used
+        $this->addTransitionAction(self::ST_QUOTED_LEXEME, self::ST_WHITE_SPACE,   $addQuotedLexemeAction);
+
+
+        /** Lexeme modifier */
+        $this->addEntryAction(self::ST_LEXEME_MODIFIER, $lexemeModifierAction);
+
+
+        /** Number */
+        $this->addEntryAction(self::ST_NUMBER,                           $addLexemeCharAction);
+        $this->addEntryAction(self::ST_MANTISSA,                         $addLexemeCharAction);
+        $this->addTransitionAction(self::ST_NUMBER,   self::ST_NUMBER,   $addLexemeCharAction);
+        // ST_NUMBER => ST_MANTISSA transition is covered by ST_MANTISSA entry action
+        $this->addTransitionAction(self::ST_MANTISSA, self::ST_MANTISSA, $addLexemeCharAction);
+
+        $this->addTransitionAction(self::ST_NUMBER,   self::ST_WHITE_SPACE,     $addNumberLexemeAction);
+        $this->addTransitionAction(self::ST_NUMBER,   self::ST_SYNT_LEXEME,     $addNumberLexemeAction);
+        $this->addTransitionAction(self::ST_NUMBER,   self::ST_LEXEME_MODIFIER, $addNumberLexemeAction);
+        $this->addTransitionAction(self::ST_MANTISSA, self::ST_WHITE_SPACE,     $addNumberLexemeAction);
+        $this->addTransitionAction(self::ST_MANTISSA, self::ST_SYNT_LEXEME,     $addNumberLexemeAction);
+        $this->addTransitionAction(self::ST_MANTISSA, self::ST_LEXEME_MODIFIER, $addNumberLexemeAction);
+    }
+
+
+
+
+    /**
+     * Translate input char to an input symbol of state machine
+     *
+     * @param string $char
+     * @return integer
+     */
+    private function _translateInput($char)
+    {
+        if        (strpos(self::QUERY_WHITE_SPACE_CHARS,    $char) !== false) { return self::IN_WHITE_SPACE;
+        } else if (strpos(self::QUERY_SYNT_CHARS,           $char) !== false) { return self::IN_SYNT_CHAR;
+        } else if (strpos(self::QUERY_MUTABLE_CHARS,        $char) !== false) { return self::IN_MUTABLE_CHAR;
+        } else if (strpos(self::QUERY_LEXEMEMODIFIER_CHARS, $char) !== false) { return self::IN_LEXEME_MODIFIER;
+        } else if (strpos(self::QUERY_ASCIIDIGITS_CHARS,    $char) !== false) { return self::IN_ASCII_DIGIT;
+        } else if ($char === '"' )                                            { return self::IN_QUOTE;
+        } else if ($char === '.' )                                            { return self::IN_DECIMAL_POINT;
+        } else if ($char === '\\')                                            { return self::IN_ESCAPE_CHAR;
+        } else                                                                { return self::IN_CHAR;
+        }
+    }
+
+
+    /**
+     * This method is used to tokenize query string into lexemes
+     *
+     * @param string $inputString
+     * @param string $encoding
+     * @return array
+     * @throws Zend_Search_Lucene_Search_QueryParserException
+     */
+    public function tokenize($inputString, $encoding)
+    {
+        $this->reset();
+
+        $this->_lexemes     = array();
+        $this->_queryString = array();
+
+        $strLength = iconv_strlen($inputString, $encoding);
+
+        // Workaround for iconv_substr bug
+        $inputString .= ' ';
+
+        for ($count = 0; $count < $strLength; $count++) {
+            $this->_queryString[$count] = iconv_substr($inputString, $count, 1, $encoding);
+        }
+
+        for ($this->_queryStringPosition = 0;
+             $this->_queryStringPosition < count($this->_queryString);
+             $this->_queryStringPosition++) {
+            $this->process($this->_translateInput($this->_queryString[$this->_queryStringPosition]));
+        }
+
+        $this->process(self::IN_WHITE_SPACE);
+
+        if ($this->getState() != self::ST_WHITE_SPACE) {
+            throw new Zend_Search_Lucene_Search_QueryParserException('Unexpected end of query');
+        }
+
+        $this->_queryString = null;
+
+        return $this->_lexemes;
+    }
+
+
+
+    /*********************************************************************
+     * Actions implementation
+     *
+     * Actions affect on recognized lexemes list
+     *********************************************************************/
+
+    /**
+     * Add query syntax lexeme
+     *
+     * @throws Zend_Search_Lucene_Search_QueryParserException
+     */
+    public function addQuerySyntaxLexeme()
+    {
+        $lexeme = $this->_queryString[$this->_queryStringPosition];
+
+        // Process two char lexemes
+        if (strpos(self::QUERY_DOUBLECHARLEXEME_CHARS, $lexeme) !== false) {
+            // increase current position in a query string
+            $this->_queryStringPosition++;
+
+            // check,
+            if ($this->_queryStringPosition == count($this->_queryString)  ||
+                $this->_queryString[$this->_queryStringPosition] != $lexeme) {
+                    throw new Zend_Search_Lucene_Search_QueryParserException('Two chars lexeme expected. ' . $this->_positionMsg());
+                }
+
+            // duplicate character
+            $lexeme .= $lexeme;
+        }
+
+        $token = new Zend_Search_Lucene_Search_QueryToken(
+                                Zend_Search_Lucene_Search_QueryToken::TC_SYNTAX_ELEMENT,
+                                $lexeme,
+                                $this->_queryStringPosition);
+
+        // Skip this lexeme if it's a field indicator ':' and treat previous as 'field' instead of 'word'
+        if ($token->type == Zend_Search_Lucene_Search_QueryToken::TT_FIELD_INDICATOR) {
+            $token = array_pop($this->_lexemes);
+            if ($token === null  ||  $token->type != Zend_Search_Lucene_Search_QueryToken::TT_WORD) {
+                throw new Zend_Search_Lucene_Search_QueryParserException('Field mark \':\' must follow field name. ' . $this->_positionMsg());
+            }
+
+            $token->type = Zend_Search_Lucene_Search_QueryToken::TT_FIELD;
+        }
+
+        $this->_lexemes[] = $token;
+    }
+
+    /**
+     * Add lexeme modifier
+     */
+    public function addLexemeModifier()
+    {
+        $this->_lexemes[] = new Zend_Search_Lucene_Search_QueryToken(
+                                    Zend_Search_Lucene_Search_QueryToken::TC_SYNTAX_ELEMENT,
+                                    $this->_queryString[$this->_queryStringPosition],
+                                    $this->_queryStringPosition);
+    }
+
+
+    /**
+     * Add lexeme
+     */
+    public function addLexeme()
+    {
+        $this->_lexemes[] = new Zend_Search_Lucene_Search_QueryToken(
+                                    Zend_Search_Lucene_Search_QueryToken::TC_WORD,
+                                    $this->_currentLexeme,
+                                    $this->_queryStringPosition - 1);
+
+        $this->_currentLexeme = '';
+    }
+
+    /**
+     * Add quoted lexeme
+     */
+    public function addQuotedLexeme()
+    {
+        $this->_lexemes[] = new Zend_Search_Lucene_Search_QueryToken(
+                                    Zend_Search_Lucene_Search_QueryToken::TC_PHRASE,
+                                    $this->_currentLexeme,
+                                    $this->_queryStringPosition);
+
+        $this->_currentLexeme = '';
+    }
+
+    /**
+     * Add number lexeme
+     */
+    public function addNumberLexeme()
+    {
+        $this->_lexemes[] = new Zend_Search_Lucene_Search_QueryToken(
+                                    Zend_Search_Lucene_Search_QueryToken::TC_NUMBER,
+                                    $this->_currentLexeme,
+                                    $this->_queryStringPosition - 1);
+        $this->_currentLexeme = '';
+    }
+
+    /**
+     * Extend lexeme by one char
+     */
+    public function addLexemeChar()
+    {
+        $this->_currentLexeme .= $this->_queryString[$this->_queryStringPosition];
+    }
+
+
+    /**
+     * Position message
+     *
+     * @return string
+     */
+    private function _positionMsg()
+    {
+        return 'Position is ' . $this->_queryStringPosition . '.';
+    }
+
+
+    /*********************************************************************
+     * Syntax errors actions
+     *********************************************************************/
+    public function lexModifierErrException()
+    {
+        throw new Zend_Search_Lucene_Search_QueryParserException('Lexeme modifier character can be followed only by number, white space or query syntax element. ' . $this->_positionMsg());
+    }
+    public function quoteWithinLexemeErrException()
+    {
+        throw new Zend_Search_Lucene_Search_QueryParserException('Quote within lexeme must be escaped by \'\\\' char. ' . $this->_positionMsg());
+    }
+    public function wrongNumberErrException()
+    {
+        throw new Zend_Search_Lucene_Search_QueryParserException('Wrong number syntax.' . $this->_positionMsg());
+    }
+}
+

search/Zend/Search/Lucene/Search/QueryParser.php

@@ -15,128 +15,507 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
-/** Zend_Search_Lucene_Search_QueryTokenizer */
-require_once 'Zend/Search/Lucene/Search/QueryTokenizer.php';
-
 /** Zend_Search_Lucene_Index_Term */
-require_once 'Zend/Search/Lucene/Index/Term.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php';
 
 /** Zend_Search_Lucene_Search_Query_Term */
-require_once 'Zend/Search/Lucene/Search/Query/Term.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Term.php';
 
 /** Zend_Search_Lucene_Search_Query_MultiTerm */
-require_once 'Zend/Search/Lucene/Search/Query/MultiTerm.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/MultiTerm.php';
+
+/** Zend_Search_Lucene_Search_Query_Boolean */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Boolean.php';
 
 /** Zend_Search_Lucene_Search_Query_Phrase */
-require_once 'Zend/Search/Lucene/Search/Query/Phrase.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Phrase.php';
 
+/** Zend_Search_Lucene_Search_Query_Empty */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Empty.php';
+
+
+/** Zend_Search_Lucene_Search_QueryLexer */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryLexer.php';
+
+/** Zend_Search_Lucene_Search_QueryParserContext */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserContext.php';
+
+
+/** Zend_Search_Lucene_FSM */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/FSM.php';
 
 /** Zend_Search_Lucene_Exception */
-require_once 'Zend/Search/Lucene/Exception.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+/** Zend_Search_Lucene_Search_QueryParserException */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
-class Zend_Search_Lucene_Search_QueryParser
+class Zend_Search_Lucene_Search_QueryParser extends Zend_Search_Lucene_FSM
 {
+    /**
+     * Parser instance
+     *
+     * @var Zend_Search_Lucene_Search_QueryParser
+     */
+    private static $_instance = null;
+
 
     /**
-     * Parses a query string, returning a Zend_Search_Lucene_Search_Query
+     * Query lexer
+     *
+     * @var Zend_Search_Lucene_Search_QueryLexer
+     */
+    private $_lexer;
+
+    /**
+     * Tokens list
+     * Array of Zend_Search_Lucene_Search_QueryToken objects
+     *
+     * @var array
+     */
+    private $_tokens;
+
+    /**
+     * Current token
+     *
+     * @var integer|string
+     */
+    private $_currentToken;
+
+    /**
+     * Last token
+     *
+     * It can be processed within FSM states, but this addirional state simplifies FSM
+     *
+     * @var Zend_Search_Lucene_Search_QueryToken
+     */
+    private $_lastToken = null;
+
+    /**
+     * Range query first term
+     *
+     * @var string
+     */
+    private $_rqFirstTerm = null;
+
+    /**
+     * Current query parser context
+     *
+     * @var Zend_Search_Lucene_Search_QueryParserContext
+     */
+    private $_context;
+
+    /**
+     * Context stack
+     *
+     * @var array
+     */
+    private $_contextStack;
+
+    /**
+     * Query string encoding
+     *
+     * @var string
+     */
+    private $_encoding;
+
+    /**
+     * Query string default encoding
+     *
+     * @var string
+     */
+    private $_defaultEncoding = '';
+
+
+    /**
+     * Boolean operators constants
+     */
+    const B_OR  = 0;
+    const B_AND = 1;
+
+    /**
+     * Default boolean queries operator
+     *
+     * @var integer
+     */
+    private $_defaultOperator = self::B_OR;
+
+
+    /** Query parser State Machine states */
+    const ST_COMMON_QUERY_ELEMENT       = 0;   // Terms, phrases, operators
+    const ST_CLOSEDINT_RQ_START         = 1;   // Range query start (closed interval) - '['
+    const ST_CLOSEDINT_RQ_FIRST_TERM    = 2;   // First term in '[term1 to term2]' construction
+    const ST_CLOSEDINT_RQ_TO_TERM       = 3;   // 'TO' lexeme in '[term1 to term2]' construction
+    const ST_CLOSEDINT_RQ_LAST_TERM     = 4;   // Second term in '[term1 to term2]' construction
+    const ST_CLOSEDINT_RQ_END           = 5;   // Range query end (closed interval) - ']'
+    const ST_OPENEDINT_RQ_START         = 6;   // Range query start (opened interval) - '{'
+    const ST_OPENEDINT_RQ_FIRST_TERM    = 7;   // First term in '{term1 to term2}' construction
+    const ST_OPENEDINT_RQ_TO_TERM       = 8;   // 'TO' lexeme in '{term1 to term2}' construction
+    const ST_OPENEDINT_RQ_LAST_TERM     = 9;   // Second term in '{term1 to term2}' construction
+    const ST_OPENEDINT_RQ_END           = 10;  // Range query end (opened interval) - '}'
+
+    /**
+     * Parser constructor
+     */
+    public function __construct()
+    {
+        parent::__construct(array(self::ST_COMMON_QUERY_ELEMENT,
+                                  self::ST_CLOSEDINT_RQ_START,
+                                  self::ST_CLOSEDINT_RQ_FIRST_TERM,
+                                  self::ST_CLOSEDINT_RQ_TO_TERM,
+                                  self::ST_CLOSEDINT_RQ_LAST_TERM,
+                                  self::ST_CLOSEDINT_RQ_END,
+                                  self::ST_OPENEDINT_RQ_START,
+                                  self::ST_OPENEDINT_RQ_FIRST_TERM,
+                                  self::ST_OPENEDINT_RQ_TO_TERM,
+                                  self::ST_OPENEDINT_RQ_LAST_TERM,
+                                  self::ST_OPENEDINT_RQ_END
+                                 ),
+                            Zend_Search_Lucene_Search_QueryToken::getTypes());
+
+        $this->addRules(
+             array(array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_WORD,             self::ST_COMMON_QUERY_ELEMENT),
+                   array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_PHRASE,           self::ST_COMMON_QUERY_ELEMENT),
+                   array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_FIELD,            self::ST_COMMON_QUERY_ELEMENT),
+                   array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_REQUIRED,         self::ST_COMMON_QUERY_ELEMENT),
+                   array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_PROHIBITED,       self::ST_COMMON_QUERY_ELEMENT),
+                   array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_FUZZY_PROX_MARK,  self::ST_COMMON_QUERY_ELEMENT),
+                   array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_BOOSTING_MARK,    self::ST_COMMON_QUERY_ELEMENT),
+                   array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_RANGE_INCL_START, self::ST_CLOSEDINT_RQ_START),
+                   array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_RANGE_EXCL_START, self::ST_OPENEDINT_RQ_START),
+                   array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_SUBQUERY_START,   self::ST_COMMON_QUERY_ELEMENT),
+                   array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_SUBQUERY_END,     self::ST_COMMON_QUERY_ELEMENT),
+                   array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_AND_LEXEME,       self::ST_COMMON_QUERY_ELEMENT),
+                   array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_OR_LEXEME,        self::ST_COMMON_QUERY_ELEMENT),
+                   array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_NOT_LEXEME,       self::ST_COMMON_QUERY_ELEMENT),
+                   array(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_NUMBER,           self::ST_COMMON_QUERY_ELEMENT)
+                  ));
+        $this->addRules(
+             array(array(self::ST_CLOSEDINT_RQ_START,      Zend_Search_Lucene_Search_QueryToken::TT_WORD,           self::ST_CLOSEDINT_RQ_FIRST_TERM),
+                   array(self::ST_CLOSEDINT_RQ_FIRST_TERM, Zend_Search_Lucene_Search_QueryToken::TT_TO_LEXEME,      self::ST_CLOSEDINT_RQ_TO_TERM),
+                   array(self::ST_CLOSEDINT_RQ_TO_TERM,    Zend_Search_Lucene_Search_QueryToken::TT_WORD,           self::ST_CLOSEDINT_RQ_LAST_TERM),
+                   array(self::ST_CLOSEDINT_RQ_LAST_TERM,  Zend_Search_Lucene_Search_QueryToken::TT_RANGE_INCL_END, self::ST_COMMON_QUERY_ELEMENT)
+                  ));
+        $this->addRules(
+             array(array(self::ST_OPENEDINT_RQ_START,      Zend_Search_Lucene_Search_QueryToken::TT_WORD,           self::ST_OPENEDINT_RQ_FIRST_TERM),
+                   array(self::ST_OPENEDINT_RQ_FIRST_TERM, Zend_Search_Lucene_Search_QueryToken::TT_TO_LEXEME,      self::ST_OPENEDINT_RQ_TO_TERM),
+                   array(self::ST_OPENEDINT_RQ_TO_TERM,    Zend_Search_Lucene_Search_QueryToken::TT_WORD,           self::ST_OPENEDINT_RQ_LAST_TERM),
+                   array(self::ST_OPENEDINT_RQ_LAST_TERM,  Zend_Search_Lucene_Search_QueryToken::TT_RANGE_EXCL_END, self::ST_COMMON_QUERY_ELEMENT)
+                  ));
+
+
+
+        $addTermEntryAction             = new Zend_Search_Lucene_FSMAction($this, 'addTermEntry');
+        $addPhraseEntryAction           = new Zend_Search_Lucene_FSMAction($this, 'addPhraseEntry');
+        $setFieldAction                 = new Zend_Search_Lucene_FSMAction($this, 'setField');
+        $setSignAction                  = new Zend_Search_Lucene_FSMAction($this, 'setSign');
+        $setFuzzyProxAction             = new Zend_Search_Lucene_FSMAction($this, 'processFuzzyProximityModifier');
+        $processModifierParameterAction = new Zend_Search_Lucene_FSMAction($this, 'processModifierParameter');
+        $subqueryStartAction            = new Zend_Search_Lucene_FSMAction($this, 'subqueryStart');
+        $subqueryEndAction              = new Zend_Search_Lucene_FSMAction($this, 'subqueryEnd');
+        $logicalOperatorAction          = new Zend_Search_Lucene_FSMAction($this, 'logicalOperator');
+        $openedRQFirstTermAction        = new Zend_Search_Lucene_FSMAction($this, 'openedRQFirstTerm');
+        $openedRQLastTermAction         = new Zend_Search_Lucene_FSMAction($this, 'openedRQLastTerm');
+        $closedRQFirstTermAction        = new Zend_Search_Lucene_FSMAction($this, 'closedRQFirstTerm');
+        $closedRQLastTermAction         = new Zend_Search_Lucene_FSMAction($this, 'closedRQLastTerm');
+
+
+        $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_WORD,            $addTermEntryAction);
+        $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_PHRASE,          $addPhraseEntryAction);
+        $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_FIELD,           $setFieldAction);
+        $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_REQUIRED,        $setSignAction);
+        $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_PROHIBITED,      $setSignAction);
+        $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_FUZZY_PROX_MARK, $setFuzzyProxAction);
+        $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_NUMBER,          $processModifierParameterAction);
+        $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_SUBQUERY_START,  $subqueryStartAction);
+        $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_SUBQUERY_END,    $subqueryEndAction);
+        $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_AND_LEXEME,      $logicalOperatorAction);
+        $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_OR_LEXEME,       $logicalOperatorAction);
+        $this->addInputAction(self::ST_COMMON_QUERY_ELEMENT, Zend_Search_Lucene_Search_QueryToken::TT_NOT_LEXEME,      $logicalOperatorAction);
+
+        $this->addEntryAction(self::ST_OPENEDINT_RQ_FIRST_TERM, $openedRQFirstTermAction);
+        $this->addEntryAction(self::ST_OPENEDINT_RQ_LAST_TERM,  $openedRQLastTermAction);
+        $this->addEntryAction(self::ST_CLOSEDINT_RQ_FIRST_TERM, $closedRQFirstTermAction);
+        $this->addEntryAction(self::ST_CLOSEDINT_RQ_LAST_TERM,  $closedRQLastTermAction);
+
+
+
+        $this->_lexer = new Zend_Search_Lucene_Search_QueryLexer();
+    }
+
+
+    /**
+     * Set query string default encoding
+     *
+     * @param string $encoding
+     */
+    public static function setDefaultEncoding($encoding)
+    {
+        if (self::$_instance === null) {
+            self::$_instance = new Zend_Search_Lucene_Search_QueryParser();
+        }
+
+        self::$_instance->_defaultEncoding = $encoding;
+    }
+
+    /**
+     * Get query string default encoding
+     *
+     * @return string
+     */
+    public static function getDefaultEncoding()
+    {
+        if (self::$_instance === null) {
+            self::$_instance = new Zend_Search_Lucene_Search_QueryParser();
+        }
+
+        return self::$_instance->_defaultEncoding;
+    }
+
+    /**
+     * Set default boolean operator
+     *
+     * @param integer $operator
+     */
+    public static function setDefaultOperator($operator)
+    {
+        if (self::$_instance === null) {
+            self::$_instance = new Zend_Search_Lucene_Search_QueryParser();
+        }
+
+        self::$_instance->_defaultOperator = $operator;
+    }
+
+    /**
+     * Get default boolean operator
+     *
+     * @return integer
+     */
+    public static function getDefaultOperator()
+    {
+        if (self::$_instance === null) {
+            self::$_instance = new Zend_Search_Lucene_Search_QueryParser();
+        }
+
+        return self::$_instance->_defaultOperator;
+    }
+
+    /**
+     * Parses a query string
      *
      * @param string $strQuery
+     * @param string $encoding
      * @return Zend_Search_Lucene_Search_Query
+     * @throws Zend_Search_Lucene_Search_QueryParserException
      */
-    static public function parse($strQuery)
+    public static function parse($strQuery, $encoding = null)
     {
-        $tokens = new Zend_Search_Lucene_Search_QueryTokenizer($strQuery);
+        if (self::$_instance === null) {
+            self::$_instance = new Zend_Search_Lucene_Search_QueryParser();
+        }
+
+        self::$_instance->_encoding     = ($encoding !== null) ? $encoding : self::$_instance->_defaultEncoding;
+        self::$_instance->_lastToken    = null;
+        self::$_instance->_context      = new Zend_Search_Lucene_Search_QueryParserContext(self::$_instance->_encoding);
+        self::$_instance->_contextStack = array();
+        self::$_instance->_tokens       = self::$_instance->_lexer->tokenize($strQuery, self::$_instance->_encoding);
 
         // Empty query
-        if (!$tokens->count()) {
-            throw new Zend_Search_Lucene_Exception('Syntax error: query string cannot be empty.');
+        if (count(self::$_instance->_tokens) == 0) {
+            return new Zend_Search_Lucene_Search_Query_Empty();
         }
 
-        // Term query
-        if ($tokens->count() == 1) {
-            if ($tokens->current()->type == Zend_Search_Lucene_Search_QueryToken::TOKTYPE_WORD) {
-                return new Zend_Search_Lucene_Search_Query_Term(new Zend_Search_Lucene_Index_Term($tokens->current()->text, 'contents'));
-            } else {
-                throw new Zend_Search_Lucene_Exception('Syntax error: query string must contain at least one word.');
+
+        foreach (self::$_instance->_tokens as $token) {
+            try {
+                self::$_instance->_currentToken = $token;
+                self::$_instance->process($token->type);
+
+                self::$_instance->_lastToken = $token;
+            } catch (Exception $e) {
+                if (strpos($e->getMessage(), 'There is no any rule for') !== false) {
+                    throw new Zend_Search_Lucene_Search_QueryParserException( 'Syntax error at char position ' . $token->position . '.' );
+                }
+
+                throw $e;
             }
         }
 
-
-        /**
-         * MultiTerm Query
-         *
-         * Process each token that was returned by the tokenizer.
-         */
-        $terms = array();
-        $signs = array();
-        $prevToken = null;
-        $openBrackets = 0;
-        $field = 'contents';
-        foreach ($tokens as $token) {
-            switch ($token->type) {
-                case Zend_Search_Lucene_Search_QueryToken::TOKTYPE_WORD:
-                    $terms[] = new Zend_Search_Lucene_Index_Term($token->text, $field);
-                    $field = 'contents';
-                    if ($prevToken !== null &&
-                        $prevToken->type == Zend_Search_Lucene_Search_QueryToken::TOKTYPE_SIGN) {
-                            if ($prevToken->text == "+") {
-                                $signs[] = true;
-                            } else {
-                                $signs[] = false;
-                            }
-                    } else {
-                        $signs[] = null;
-                    }
-                    break;
-                case Zend_Search_Lucene_Search_QueryToken::TOKTYPE_SIGN:
-                    if ($prevToken !== null &&
-                        $prevToken->type == Zend_Search_Lucene_Search_QueryToken::TOKTYPE_SIGN) {
-                            throw new Zend_Search_Lucene_Exception('Syntax error: sign operator must be followed by a word.');
-                    }
-                    break;
-                case Zend_Search_Lucene_Search_QueryToken::TOKTYPE_FIELD:
-                    $field = $token->text;
-                    // let previous token to be signed as next $prevToken
-                    $token = $prevToken;
-                    break;
-                case Zend_Search_Lucene_Search_QueryToken::TOKTYPE_BRACKET:
-                    $token->text=='(' ? $openBrackets++ : $openBrackets--;
-            }
-            $prevToken = $token;
+        if (count(self::$_instance->_contextStack) != 0) {
+            throw new Zend_Search_Lucene_Search_QueryParserException('Syntax Error: mismatched parentheses, every opening must have closing.' );
         }
 
-        // Finish up parsing: check the last token in the query for an opening sign or parenthesis.
-        if ($prevToken->type == Zend_Search_Lucene_Search_QueryToken::TOKTYPE_SIGN) {
-            throw new Zend_Search_Lucene_Exception('Syntax Error: sign operator must be followed by a word.');
+        return self::$_instance->_context->getQuery();
+    }
+
+
+    /*********************************************************************
+     * Actions implementation
+     *
+     * Actions affect on recognized lexemes list
+     *********************************************************************/
+
+    /**
+     * Add term to a query
+     */
+    public function addTermEntry()
+    {
+        $entry = new Zend_Search_Lucene_Search_QueryEntry_Term($this->_currentToken->text, $this->_context->getField());
+        $this->_context->addEntry($entry);
+    }
+
+    /**
+     * Add phrase to a query
+     */
+    public function addPhraseEntry()
+    {
+        $entry = new Zend_Search_Lucene_Search_QueryEntry_Phrase($this->_currentToken->text, $this->_context->getField());
+        $this->_context->addEntry($entry);
+    }
+
+    /**
+     * Set entry field
+     */
+    public function setField()
+    {
+        $this->_context->setNextEntryField($this->_currentToken->text);
+    }
+
+    /**
+     * Set entry sign
+     */
+    public function setSign()
+    {
+        $this->_context->setNextEntrySign($this->_currentToken->type);
+    }
+
+
+    /**
+     * Process fuzzy search/proximity modifier - '~'
+     */
+    public function processFuzzyProximityModifier()
+    {
+        $this->_context->processFuzzyProximityModifier();
+    }
+
+    /**
+     * Process modifier parameter
+     *
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public function processModifierParameter()
+    {
+        if ($this->_lastToken === null) {
+            throw new Zend_Search_Lucene_Search_QueryParserException('Lexeme modifier parameter must follow lexeme modifier. Char position 0.' );
         }
 
-        // Finish up parsing: check that every opening bracket has a matching closing bracket.
-        if ($openBrackets != 0) {
-            throw new Zend_Search_Lucene_Exception('Syntax Error: mismatched parentheses, every opening must have closing.');
-        }
+        switch ($this->_lastToken->type) {
+            case Zend_Search_Lucene_Search_QueryToken::TT_FUZZY_PROX_MARK:
+                $this->_context->processFuzzyProximityModifier($this->_currentToken->text);
+                break;
+
+            case Zend_Search_Lucene_Search_QueryToken::TT_BOOSTING_MARK:
+                $this->_context->boost($this->_currentToken->text);
+                break;
 
-        switch (count($terms)) {
-            case 0:
-                throw new Zend_Search_Lucene_Exception('Syntax error: bad term count.');
-            case 1:
-                return new Zend_Search_Lucene_Search_Query_Term($terms[0],$signs[0] !== false);
             default:
-                return new Zend_Search_Lucene_Search_Query_MultiTerm($terms,$signs);
+                // It's not a user input exception
+                throw new Zend_Search_Lucene_Exception('Lexeme modifier parameter must follow lexeme modifier. Char position .' );
         }
     }
 
+
+    /**
+     * Start subquery
+     */
+    public function subqueryStart()
+    {
+        $this->_contextStack[] = $this->_context;
+        $this->_context        = new Zend_Search_Lucene_Search_QueryParserContext($this->_encoding, $this->_context->getField());
+    }
+
+    /**
+     * End subquery
+     */
+    public function subqueryEnd()
+    {
+        if (count($this->_contextStack) == 0) {
+            throw new Zend_Search_Lucene_Search_QueryParserException('Syntax Error: mismatched parentheses, every opening must have closing. Char position ' . $this->_currentToken->position . '.' );
+        }
+
+        $query          = $this->_context->getQuery();
+        $this->_context = array_pop($this->_contextStack);
+
+        $this->_context->addEntry(new Zend_Search_Lucene_Search_QueryEntry_Subquery($query));
+    }
+
+    /**
+     * Process logical operator
+     */
+    public function logicalOperator()
+    {
+        $this->_context->addLogicalOperator($this->_currentToken->type);
+    }
+
+    /**
+     * Process first range query term (opened interval)
+     */
+    public function openedRQFirstTerm()
+    {
+        $this->_rqFirstTerm = $this->_currentToken->text;
+    }
+
+    /**
+     * Process last range query term (opened interval)
+     *
+     * @throws Zend_Search_Lucene_Search_QueryParserException
+     */
+    public function openedRQLastTerm()
+    {
+        throw new Zend_Search_Lucene_Search_QueryParserException('Range queries are not supported yet.');
+
+        // $firstTerm = new Zend_Search_Lucene_Index_Term($this->_rqFirstTerm,        $this->_context->getField());
+        // $lastTerm  = new Zend_Search_Lucene_Index_Term($this->_currentToken->text, $this->_context->getField());
+
+        // $query = new Zend_Search_Lucene_Search_Query_Range($firstTerm, $lastTerm, false);
+        // $this->_context->addentry($query);
+    }
+
+    /**
+     * Process first range query term (closed interval)
+     */
+    public function closedRQFirstTerm()
+    {
+        $this->_rqFirstTerm = $this->_currentToken->text;
+    }
+
+    /**
+     * Process last range query term (closed interval)
+     *
+     * @throws Zend_Search_Lucene_Search_QueryParserException
+     */
+    public function closedRQLastTerm()
+    {
+        throw new Zend_Search_Lucene_Search_QueryParserException('Range queries are not supported yet.');
+
+        // $firstTerm = new Zend_Search_Lucene_Index_Term($this->_rqFirstTerm,        $this->_context->getField());
+        // $lastTerm  = new Zend_Search_Lucene_Index_Term($this->_currentToken->text, $this->_context->getField());
+
+        // $query = new Zend_Search_Lucene_Search_Query_Range($firstTerm, $lastTerm, true);
+        // $this->_context->addentry($query);
+    }
 }
 

search/Zend/Search/Lucene/Search/QueryParserContext.php

@@ -0,0 +1,416 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+/** Zend_Search_Lucene_FSM */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/FSM.php';
+
+
+/** Zend_Search_Lucene_Index_Term */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php';
+
+/** Zend_Search_Lucene_Search_QueryToken */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryToken.php';
+
+/** Zend_Search_Lucene_Search_Query_Term */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Term.php';
+
+/** Zend_Search_Lucene_Search_Query_MultiTerm */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/MultiTerm.php';
+
+/** Zend_Search_Lucene_Search_Query_Boolean */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Boolean.php';
+
+/** Zend_Search_Lucene_Search_Query_Phrase */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Phrase.php';
+
+/** Zend_Search_Lucene_Exception */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+/** Zend_Search_Lucene_Search_QueryParserException */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php';
+
+/** Zend_Search_Lucene_Search_BooleanExpressionRecognizer */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/BooleanExpressionRecognizer.php';
+
+/** Zend_Search_Lucene_Search_QueryEntry */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Search_QueryParserContext
+{
+    /**
+     * Default field for the context.
+     *
+     * null means, that term should be searched through all fields
+     * Zend_Search_Lucene_Search_Query::rewriteQuery($index) transletes such queries to several
+     *
+     * @var string|null
+     */
+    private $_defaultField;
+
+    /**
+     * Field specified for next entry
+     *
+     * @var string
+     */
+    private $_nextEntryField = null;
+
+    /**
+     * True means, that term is required.
+     * False means, that term is prohibited.
+     * null means, that term is neither prohibited, nor required
+     *
+     * @var boolean
+     */
+    private $_nextEntrySign = null;
+
+
+    /**
+     * Entries grouping mode
+     */
+    const GM_SIGNS   = 0;  // Signs mode: '+term1 term2 -term3 +(subquery1) -(subquery2)'
+    const GM_BOOLEAN = 1;  // Boolean operators mode: 'term1 and term2  or  (subquery1) and not (subquery2)'
+
+    /**
+     * Grouping mode
+     *
+     * @var integer
+     */
+    private $_mode = null;
+
+    /**
+     * Entries signs.
+     * Used in GM_SIGNS grouping mode
+     *
+     * @var arrays
+     */
+    private $_signs = array();
+
+    /**
+     * Query entries
+     * Each entry is a Zend_Search_Lucene_Search_QueryEntry object or
+     * boolean operator (Zend_Search_Lucene_Search_QueryToken class constant)
+     *
+     * @var array
+     */
+    private $_entries = array();
+
+    /**
+     * Query string encoding
+     *
+     * @var string
+     */
+    private $_encoding;
+
+
+    /**
+     * Context object constructor
+     *
+     * @param string $encoding
+     * @param string|null $defaultField
+     */
+    public function __construct($encoding, $defaultField = null)
+    {
+        $this->_encoding     = $encoding;
+        $this->_defaultField = $defaultField;
+    }
+
+
+    /**
+     * Get context default field
+     *
+     * @return string|null
+     */
+    public function getField()
+    {
+        return ($this->_nextEntryField !== null)  ?  $this->_nextEntryField : $this->_defaultField;
+    }
+
+    /**
+     * Set field for next entry
+     *
+     * @param string $field
+     */
+    public function setNextEntryField($field)
+    {
+        $this->_nextEntryField = $field;
+    }
+
+
+    /**
+     * Set sign for next entry
+     *
+     * @param integer $sign
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public function setNextEntrySign($sign)
+    {
+        if ($this->_mode === self::GM_BOOLEAN) {
+            throw new Zend_Search_Lucene_Search_QueryParserException('It\'s not allowed to mix boolean and signs styles in the same subquery.');
+        }
+
+        $this->_mode = self::GM_SIGNS;
+
+        if ($sign == Zend_Search_Lucene_Search_QueryToken::TT_REQUIRED) {
+            $this->_nextEntrySign = true;
+        } else if ($sign == Zend_Search_Lucene_Search_QueryToken::TT_PROHIBITED) {
+            $this->_nextEntrySign = false;
+        } else {
+            throw new Zend_Search_Lucene_Exception('Unrecognized sign type.');
+        }
+    }
+
+
+    /**
+     * Add entry to a query
+     *
+     * @param Zend_Search_Lucene_Search_QueryEntry $entry
+     */
+    public function addEntry(Zend_Search_Lucene_Search_QueryEntry $entry)
+    {
+        if ($this->_mode !== self::GM_BOOLEAN) {
+            $this->_signs[] = $this->_nextEntrySign;
+        }
+
+        $this->_entries[] = $entry;
+
+        $this->_nextEntryField = null;
+        $this->_nextEntrySign  = null;
+    }
+
+
+    /**
+     * Process fuzzy search or proximity search modifier
+     *
+     * @throws Zend_Search_Lucene_Search_QueryParserException
+     */
+    public function processFuzzyProximityModifier($parameter = null)
+    {
+        // Check, that modifier has came just after word or phrase
+        if ($this->_nextEntryField !== null  ||  $this->_nextEntrySign !== null) {
+            throw new Zend_Search_Lucene_Search_QueryParserException('\'~\' modifier must follow word or phrase.');
+        }
+
+        $lastEntry = array_pop($this->_entries);
+
+        if (!$lastEntry instanceof Zend_Search_Lucene_Search_QueryEntry) {
+            // there are no entries or last entry is boolean operator
+            throw new Zend_Search_Lucene_Search_QueryParserException('\'~\' modifier must follow word or phrase.');
+        }
+
+        $lastEntry->processFuzzyProximityModifier($parameter);
+
+        $this->_entries[] = $lastEntry;
+    }
+
+    /**
+     * Set boost factor to the entry
+     *
+     * @param float $boostFactor
+     */
+    public function boost($boostFactor)
+    {
+        // Check, that modifier has came just after word or phrase
+        if ($this->_nextEntryField !== null  ||  $this->_nextEntrySign !== null) {
+            throw new Zend_Search_Lucene_Search_QueryParserException('\'^\' modifier must follow word, phrase or subquery.');
+        }
+
+        $lastEntry = array_pop($this->_entries);
+
+        if (!$lastEntry instanceof Zend_Search_Lucene_Search_QueryEntry) {
+            // there are no entries or last entry is boolean operator
+            throw new Zend_Search_Lucene_Search_QueryParserException('\'^\' modifier must follow word, phrase or subquery.');
+        }
+
+        $lastEntry->boost($boostFactor);
+
+        $this->_entries[] = $lastEntry;
+    }
+
+    /**
+     * Process logical operator
+     *
+     * @param integer $operator
+     */
+    public function addLogicalOperator($operator)
+    {
+        if ($this->_mode === self::GM_SIGNS) {
+            throw new Zend_Search_Lucene_Search_QueryParserException('It\'s not allowed to mix boolean and signs styles in the same subquery.');
+        }
+
+        $this->_mode = self::GM_BOOLEAN;
+
+        $this->_entries[] = $operator;
+    }
+
+
+    /**
+     * Generate 'signs style' query from the context
+     * '+term1 term2 -term3 +(<subquery1>) ...'
+     *
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    public function _signStyleExpressionQuery()
+    {
+        $query = new Zend_Search_Lucene_Search_Query_Boolean();
+
+        if (Zend_Search_Lucene_Search_QueryParser::getDefaultOperator() == Zend_Search_Lucene_Search_QueryParser::B_AND) {
+            $defaultSign = true; // required
+        } else {
+            // Zend_Search_Lucene_Search_QueryParser::B_OR
+            $defaultSign = null; // optional
+        }
+
+        foreach ($this->_entries as $entryId => $entry) {
+            $sign = ($this->_signs[$entryId] !== null) ?  $this->_signs[$entryId] : $defaultSign;
+            $query->addSubquery($entry->getQuery($this->_encoding), $sign);
+        }
+
+        return $query;
+    }
+
+
+    /**
+     * Generate 'boolean style' query from the context
+     * 'term1 and term2   or   term3 and (<subquery1>) and not (<subquery2>)'
+     *
+     * @return Zend_Search_Lucene_Search_Query
+     * @throws Zend_Search_Lucene
+     */
+    private function _booleanExpressionQuery()
+    {
+        /**
+         * We treat each level of an expression as a boolean expression in
+         * a Disjunctive Normal Form
+         *
+         * AND operator has higher precedence than OR
+         *
+         * Thus logical query is a disjunction of one or more conjunctions of
+         * one or more query entries
+         */
+
+        $expressionRecognizer = new Zend_Search_Lucene_Search_BooleanExpressionRecognizer();
+
+        try {
+            foreach ($this->_entries as $entry) {
+                if ($entry instanceof Zend_Search_Lucene_Search_QueryEntry) {
+                    $expressionRecognizer->processLiteral($entry);
+                } else {
+                    switch ($entry) {
+                        case Zend_Search_Lucene_Search_QueryToken::TT_AND_LEXEME:
+                            $expressionRecognizer->processOperator(Zend_Search_Lucene_Search_BooleanExpressionRecognizer::IN_AND_OPERATOR);
+                            break;
+
+                        case Zend_Search_Lucene_Search_QueryToken::TT_OR_LEXEME:
+                            $expressionRecognizer->processOperator(Zend_Search_Lucene_Search_BooleanExpressionRecognizer::IN_OR_OPERATOR);
+                            break;
+
+                        case Zend_Search_Lucene_Search_QueryToken::TT_NOT_LEXEME:
+                            $expressionRecognizer->processOperator(Zend_Search_Lucene_Search_BooleanExpressionRecognizer::IN_NOT_OPERATOR);
+                            break;
+
+                        default:
+                            throw new Zend_Search_Lucene('Boolean expression error. Unknown operator type.');
+                    }
+                }
+            }
+
+            $conjuctions = $expressionRecognizer->finishExpression();
+        } catch (Zend_Search_Exception $e) {
+            // throw new Zend_Search_Lucene_Search_QueryParserException('Boolean expression error. Error message: \'' .
+            //                                                          $e->getMessage() . '\'.' );
+            // It's query syntax error message and it should be user friendly. So FSM message is omitted
+            throw new Zend_Search_Lucene_Search_QueryParserException('Boolean expression error.');
+        }
+
+        // Remove 'only negative' conjunctions
+        foreach ($conjuctions as $conjuctionId => $conjuction) {
+            $nonNegativeEntryFound = false;
+
+            foreach ($conjuction as $conjuctionEntry) {
+                if ($conjuctionEntry[1]) {
+                    $nonNegativeEntryFound = true;
+                    break;
+                }
+            }
+
+            if (!$nonNegativeEntryFound) {
+            	unset($conjuctions[$conjuctionId]);
+            }
+        }
+
+
+        $subqueries = array();
+        foreach ($conjuctions as  $conjuction) {
+            // Check, if it's a one term conjuction
+            if (count($conjuction) == 1) {
+                $subqueries[] = $conjuction[0][0]->getQuery($this->_encoding);
+            } else {
+                $subquery = new Zend_Search_Lucene_Search_Query_Boolean();
+
+                foreach ($conjuction as $conjuctionEntry) {
+                    $subquery->addSubquery($conjuctionEntry[0]->getQuery($this->_encoding), $conjuctionEntry[1]);
+                }
+
+                $subqueries[] = $subquery;
+            }
+        }
+
+        if (count($subqueries) == 0) {
+            return new Zend_Search_Lucene_Search_Query_Empty();
+        }
+
+        if (count($subqueries) == 1) {
+            return $subqueries[0];
+        }
+
+
+        $query = new Zend_Search_Lucene_Search_Query_Boolean();
+
+        foreach ($subqueries as $subquery) {
+            // Non-requirered entry/subquery
+            $query->addSubquery($subquery);
+        }
+
+        return $query;
+    }
+
+    /**
+     * Generate query from current context
+     *
+     * @return Zend_Search_Lucene_Search_Query
+     */
+    public function getQuery()
+    {
+        if ($this->_mode === self::GM_BOOLEAN) {
+            return $this->_booleanExpressionQuery();
+        } else {
+            return $this->_signStyleExpressionQuery();
+        }
+    }
+}

search/Zend/Search/Lucene/Search/QueryParserException.php

@@ -0,0 +1,40 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/**
+ * Zend_Search_Lucene base exception
+ */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ *
+ * Special exception type, which may be used to intercept wrong user input
+ */
+class Zend_Search_Lucene_Search_QueryParserException extends Zend_Search_Lucene_Exception
+{}
+

search/Zend/Search/Lucene/Search/QueryToken.php

@@ -15,46 +15,86 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Exception */
-require_once 'Zend/Search/Lucene/Exception.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Search_QueryToken
 {
     /**
-     * Token type Word.
+     * Token types.
      */
-    const TOKTYPE_WORD = 0;
+    const TT_WORD                 = 0;  // Word
+    const TT_PHRASE               = 1;  // Phrase (one or several quoted words)
+    const TT_FIELD                = 2;  // Field name in 'field:word', field:<phrase> or field:(<subquery>) pairs
+    const TT_FIELD_INDICATOR      = 3;  // ':'
+    const TT_REQUIRED             = 4;  // '+'
+    const TT_PROHIBITED           = 5;  // '-'
+    const TT_FUZZY_PROX_MARK      = 6;  // '~'
+    const TT_BOOSTING_MARK        = 7;  // '^'
+    const TT_RANGE_INCL_START     = 8;  // '['
+    const TT_RANGE_INCL_END       = 9;  // ']'
+    const TT_RANGE_EXCL_START     = 10; // '{'
+    const TT_RANGE_EXCL_END       = 11; // '}'
+    const TT_SUBQUERY_START       = 12; // '('
+    const TT_SUBQUERY_END         = 13; // ')'
+    const TT_AND_LEXEME           = 14; // 'AND' or 'and'
+    const TT_OR_LEXEME            = 15; // 'OR'  or 'or'
+    const TT_NOT_LEXEME           = 16; // 'NOT' or 'not'
+    const TT_TO_LEXEME            = 17; // 'TO'  or 'to'
+    const TT_NUMBER               = 18; // Number, like: 10, 0.8, .64, ....
+
 
     /**
-     * Token type Field.
-     * Field indicator in 'field:word' pair
+     * Returns all possible lexeme types.
+     * It's used for syntax analyzer state machine initialization
+     *
+     * @return array
      */
-    const TOKTYPE_FIELD = 1;
+    public static function getTypes()
+    {
+        return array(   self::TT_WORD,
+                        self::TT_PHRASE,
+                        self::TT_FIELD,
+                        self::TT_FIELD_INDICATOR,
+                        self::TT_REQUIRED,
+                        self::TT_PROHIBITED,
+                        self::TT_FUZZY_PROX_MARK,
+                        self::TT_BOOSTING_MARK,
+                        self::TT_RANGE_INCL_START,
+                        self::TT_RANGE_INCL_END,
+                        self::TT_RANGE_EXCL_START,
+                        self::TT_RANGE_EXCL_END,
+                        self::TT_SUBQUERY_START,
+                        self::TT_SUBQUERY_END,
+                        self::TT_AND_LEXEME,
+                        self::TT_OR_LEXEME,
+                        self::TT_NOT_LEXEME,
+                        self::TT_TO_LEXEME,
+                        self::TT_NUMBER
+                     );
+    }
+
 
     /**
-     * Token type Sign.
-     * '+' (required) or '-' (absentee) sign
+     * TokenCategories
      */
-    const TOKTYPE_SIGN = 2;
-
-    /**
-     * Token type Bracket.
-     * '(' or ')'
-     */
-    const TOKTYPE_BRACKET = 3;
+    const TC_WORD           = 0;   // Word
+    const TC_PHRASE         = 1;   // Phrase (one or several quoted words)
+    const TC_NUMBER         = 2;   // Nubers, which are used with syntax elements. Ex. roam~0.8
+    const TC_SYNTAX_ELEMENT = 3;   // +  -  ( )  [ ]  { }  !  ||  && ~ ^
 
 
     /**
@@ -71,34 +111,118 @@ class Zend_Search_Lucene_Search_QueryToken
      */
     public $text;
 
+    /**
+     * Token position within query.
+     *
+     * @var integer
+     */
+    public $position;
+
 
     /**
      * IndexReader constructor needs token type and token text as a parameters.
      *
-     * @param $tokType integer
-     * @param $tokText string
+     * @param integer $tokenCategory
+     * @param string  $tokText
+     * @param integer $position
      */
-    public function __construct($tokType, $tokText)
+    public function __construct($tokenCategory, $tokenText, $position)
     {
-        switch ($tokType) {
-            case self::TOKTYPE_BRACKET:
-                // fall through to the next case
-            case self::TOKTYPE_FIELD:
-                // fall through to the next case
-            case self::TOKTYPE_SIGN:
-                // fall through to the next case
-            case self::TOKTYPE_WORD:
+        $this->text     = $tokenText;
+        $this->position = $position + 1; // Start from 1
+
+        switch ($tokenCategory) {
+            case self::TC_WORD:
+                if (  strtolower($tokenText) == 'and') {
+                    $this->type = self::TT_AND_LEXEME;
+                } else if (strtolower($tokenText) == 'or') {
+                    $this->type = self::TT_OR_LEXEME;
+                } else if (strtolower($tokenText) == 'not') {
+                    $this->type = self::TT_NOT_LEXEME;
+                } else if (strtolower($tokenText) == 'to') {
+                    $this->type = self::TT_TO_LEXEME;
+                } else {
+                    $this->type = self::TT_WORD;
+                }
                 break;
+
+            case self::TC_PHRASE:
+                $this->type = self::TT_PHRASE;
+                break;
+
+            case self::TC_NUMBER:
+                $this->type = self::TT_NUMBER;
+                break;
+
+            case self::TC_SYNTAX_ELEMENT:
+                switch ($tokenText) {
+                    case ':':
+                        $this->type = self::TT_FIELD_INDICATOR;
+                        break;
+
+                    case '+':
+                        $this->type = self::TT_REQUIRED;
+                        break;
+
+                    case '-':
+                        $this->type = self::TT_PROHIBITED;
+                        break;
+
+                    case '~':
+                        $this->type = self::TT_FUZZY_PROX_MARK;
+                        break;
+
+                    case '^':
+                        $this->type = self::TT_BOOSTING_MARK;
+                        break;
+
+                    case '[':
+                        $this->type = self::TT_RANGE_INCL_START;
+                        break;
+
+                    case ']':
+                        $this->type = self::TT_RANGE_INCL_END;
+                        break;
+
+                    case '{':
+                        $this->type = self::TT_RANGE_EXCL_START;
+                        break;
+
+                    case '}':
+                        $this->type = self::TT_RANGE_EXCL_END;
+                        break;
+
+                    case '(':
+                        $this->type = self::TT_SUBQUERY_START;
+                        break;
+
+                    case ')':
+                        $this->type = self::TT_SUBQUERY_END;
+                        break;
+
+                    case '!':
+                        $this->type = self::TT_NOT_LEXEME;
+                        break;
+
+                    case '&&':
+                        $this->type = self::TT_AND_LEXEME;
+                        break;
+
+                    case '||':
+                        $this->type = self::TT_OR_LEXEME;
+                        break;
+
+                    default:
+                        throw new Zend_Search_Lucene_Exception('Unrecognized query syntax lexeme: \'' . $tokenText . '\'');
+                }
+                break;
+
+            case self::TC_NUMBER:
+                $this->type = self::TT_NUMBER;
+
             default:
-                throw new Zend_Search_Lucene_Exception("Unrecognized token type \"$tokType\".");
+                throw new Zend_Search_Lucene_Exception('Unrecognized lexeme type: \'' . $tokenCategory . '\'');
         }
-
-        if (!strlen($tokText)) {
-            throw new Zend_Search_Lucene_Exception('Token text must be supplied.');
-        }
-
-        $this->type = $tokType;
-        $this->text = $tokText;
     }
 }
 

search/Zend/Search/Lucene/Search/QueryTokenizer.php

@@ -21,10 +21,10 @@
 
 
 /** Zend_Search_Lucene_Search_QueryToken */
-require_once 'Zend/Search/Lucene/Search/QueryToken.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryToken.php';
 
 /** Zend_Search_Lucene_Exception */
-require_once 'Zend/Search/Lucene/Exception.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
 
 
 /**

search/Zend/Search/Lucene/Search/Similarity.php

@@ -15,20 +15,20 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Search_Similarity_Default */
-require_once 'Zend/Search/Lucene/Search/Similarity/Default.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Similarity/Default.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 abstract class Zend_Search_Lucene_Search_Similarity
@@ -38,7 +38,7 @@ abstract class Zend_Search_Lucene_Search_Similarity
      *
      * @var Zend_Search_Lucene_Search_Similarity
      */
-    static private $_defaultImpl;
+    private static $_defaultImpl;
 
     /**
      * Cache of decoded bytes.
@@ -46,7 +46,7 @@ abstract class Zend_Search_Lucene_Search_Similarity
      *
      * @var array
      */
-    static private $_normTable = array( 0   => 0.0,
+    private static $_normTable = array( 0   => 0.0,
                                         1   => 5.820766E-10,
                                         2   => 6.9849193E-10,
                                         3   => 8.1490725E-10,
@@ -310,7 +310,7 @@ abstract class Zend_Search_Lucene_Search_Similarity
      *
      * @param Zend_Search_Lucene_Search_Similarity $similarity
      */
-    static public function setDefault(Zend_Search_Lucene_Search_Similarity $similarity)
+    public static function setDefault(Zend_Search_Lucene_Search_Similarity $similarity)
     {
         self::$_defaultImpl = $similarity;
     }
@@ -322,7 +322,7 @@ abstract class Zend_Search_Lucene_Search_Similarity
      *
      * @return Zend_Search_Lucene_Search_Similarity
      */
-    static public function getDefault()
+    public static function getDefault()
     {
         if (!self::$_defaultImpl instanceof Zend_Search_Lucene_Search_Similarity) {
             self::$_defaultImpl = new Zend_Search_Lucene_Search_Similarity_Default();
@@ -381,7 +381,7 @@ abstract class Zend_Search_Lucene_Search_Similarity
      * @param integer $byte
      * @return float
      */
-    static public function decodeNorm($byte)
+    public static function decodeNorm($byte)
     {
         return self::$_normTable[$byte & 0xFF];
     }
@@ -412,7 +412,7 @@ abstract class Zend_Search_Lucene_Search_Similarity
      * @param integer $b
      * @return float
      */
-    static private function _floatToByte($f)
+    private static function _floatToByte($f)
     {
         // round negatives up to zero
         if ($f <= 0.0) {
@@ -495,10 +495,10 @@ abstract class Zend_Search_Lucene_Search_Similarity
      * Returns a score factor for the term
      *
      * @param mixed $input
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface $reader
      * @return a score factor for the term
      */
-    public function idf($input, $reader)
+    public function idf($input, Zend_Search_Lucene_Interface $reader)
     {
         if (!is_array($input)) {
             return $this->idfFreq($reader->docFreq($input), $reader->count());

search/Zend/Search/Lucene/Search/Similarity/Default.php

@@ -15,16 +15,20 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
+/** Zend_Search_Lucene_Search_Similarity */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Similarity.php';
+
+
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Search_Similarity_Default extends Zend_Search_Lucene_Search_Similarity

search/Zend/Search/Lucene/Search/Weight.php

@@ -15,7 +15,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
@@ -32,17 +32,40 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 abstract class Zend_Search_Lucene_Search_Weight
 {
+    /**
+     * Normalization factor.
+     * This value is stored only for query expanation purpose and not used in any other place
+     *
+     * @var float
+     */
+    protected $_queryNorm;
+
+    /**
+     * Weight value
+     *
+     * Weight value may be initialized in sumOfSquaredWeights() or normalize()
+     * because they both are invoked either in Query::_initWeight (for top-level query) or
+     * in corresponding methods of parent query's weights
+     *
+     * @var float
+     */
+    protected $_value;
+
+
     /**
      * The weight for this query.
      *
      * @return float
      */
-    abstract public function getValue();
+    public function getValue()
+    {
+        return $this->_value;
+    }
 
     /**
      * The sum of squared weights of contained query clauses.

search/Zend/Search/Lucene/Search/Weight/Boolean.php

@@ -0,0 +1,136 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Search_Weight */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Search_Weight_Boolean extends Zend_Search_Lucene_Search_Weight
+{
+    /**
+     * IndexReader.
+     *
+     * @var Zend_Search_Lucene_Interface
+     */
+    private $_reader;
+
+    /**
+     * The query that this concerns.
+     *
+     * @var Zend_Search_Lucene_Search_Query
+     */
+    private $_query;
+
+    /**
+     * Queries weights
+     * Array of Zend_Search_Lucene_Search_Weight
+     *
+     * @var array
+     */
+    private $_weights;
+
+
+    /**
+     * Zend_Search_Lucene_Search_Weight_Boolean constructor
+     * query - the query that this concerns.
+     * reader - index reader
+     *
+     * @param Zend_Search_Lucene_Search_Query $query
+     * @param Zend_Search_Lucene_Interface    $reader
+     */
+    public function __construct(Zend_Search_Lucene_Search_Query $query,
+                                Zend_Search_Lucene_Interface    $reader)
+    {
+        $this->_query   = $query;
+        $this->_reader  = $reader;
+        $this->_weights = array();
+
+        $signs = $query->getSigns();
+
+        foreach ($query->getSubqueries() as $num => $subquery) {
+            if ($signs === null || $signs[$num] === null || $signs[$num]) {
+                $this->_weights[$num] = $subquery->createWeight($reader);
+            }
+        }
+    }
+
+
+    /**
+     * The weight for this query
+     * Standard Weight::$_value is not used for boolean queries
+     *
+     * @return float
+     */
+    public function getValue()
+    {
+        return $this->_query->getBoost();
+    }
+
+
+    /**
+     * The sum of squared weights of contained query clauses.
+     *
+     * @return float
+     */
+    public function sumOfSquaredWeights()
+    {
+        $sum = 0;
+        foreach ($this->_weights as $weight) {
+            // sum sub weights
+            $sum += $weight->sumOfSquaredWeights();
+        }
+
+        // boost each sub-weight
+        $sum *= $this->_query->getBoost() * $this->_query->getBoost();
+
+        // check for empty query (like '-something -another')
+        if ($sum == 0) {
+            $sum = 1.0;
+        }
+        return $sum;
+    }
+
+
+    /**
+     * Assigns the query normalization factor to this.
+     *
+     * @param float $queryNorm
+     */
+    public function normalize($queryNorm)
+    {
+        // incorporate boost
+        $queryNorm *= $this->_query->getBoost();
+
+        foreach ($this->_weights as $weight) {
+            $weight->normalize($queryNorm);
+        }
+    }
+}
+
+

search/Zend/Search/Lucene/Search/Weight/Empty.php

@@ -0,0 +1,56 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Search_Weight */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Search
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Search_Weight_Empty extends Zend_Search_Lucene_Search_Weight
+{
+    /**
+     * The sum of squared weights of contained query clauses.
+     *
+     * @return float
+     */
+    public function sumOfSquaredWeights()
+    {
+        return 1;
+    }
+
+
+    /**
+     * Assigns the query normalization factor to this.
+     *
+     * @param float $queryNorm
+     */
+    public function normalize($queryNorm)
+    {
+    }
+}
+

search/Zend/Search/Lucene/Search/Weight/MultiTerm.php

@@ -15,20 +15,20 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Search_Weight */
-require_once 'Zend/Search/Lucene/Search/Weight.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Search_Weight_MultiTerm extends Zend_Search_Lucene_Search_Weight
@@ -36,14 +36,14 @@ class Zend_Search_Lucene_Search_Weight_MultiTerm extends Zend_Search_Lucene_Sear
     /**
      * IndexReader.
      *
-     * @var Zend_Search_Lucene
+     * @var Zend_Search_Lucene_Interface
      */
     private $_reader;
 
     /**
      * The query that this concerns.
      *
-     * @var Zend_Search_Lucene_Search_Query_MultiTerm
+     * @var Zend_Search_Lucene_Search_Query
      */
     private $_query;
 
@@ -61,10 +61,11 @@ class Zend_Search_Lucene_Search_Weight_MultiTerm extends Zend_Search_Lucene_Sear
      * query - the query that this concerns.
      * reader - index reader
      *
-     * @param Zend_Search_Lucene_Search_Query_MultiTerm $query
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Search_Query $query
+     * @param Zend_Search_Lucene_Interface    $reader
      */
-    public function __construct($query, $reader)
+    public function __construct(Zend_Search_Lucene_Search_Query $query,
+                                Zend_Search_Lucene_Interface    $reader)
     {
         $this->_query   = $query;
         $this->_reader  = $reader;
@@ -72,10 +73,10 @@ class Zend_Search_Lucene_Search_Weight_MultiTerm extends Zend_Search_Lucene_Sear
 
         $signs = $query->getSigns();
 
-        foreach ($query->getTerms() as $num => $term) {
-            if ($signs === null || $signs[$num] === null || $signs[$num]) {
-                $this->_weights[$num] = new Zend_Search_Lucene_Search_Weight_Term($term, $query, $reader);
-                $query->setWeight($num, $this->_weights[$num]);
+        foreach ($query->getTerms() as $id => $term) {
+            if ($signs === null || $signs[$id] === null || $signs[$id]) {
+                $this->_weights[$id] = new Zend_Search_Lucene_Search_Weight_Term($term, $query, $reader);
+                $query->setWeight($id, $this->_weights[$id]);
             }
         }
     }
@@ -83,6 +84,7 @@ class Zend_Search_Lucene_Search_Weight_MultiTerm extends Zend_Search_Lucene_Sear
 
     /**
      * The weight for this query
+     * Standard Weight::$_value is not used for boolean queries
      *
      * @return float
      */

search/Zend/Search/Lucene/Search/Weight/Phrase.php

@@ -15,7 +15,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
@@ -23,14 +23,14 @@
 /**
  * Zend_Search_Lucene_Search_Weight
  */
-require_once 'Zend/Search/Lucene/Search/Weight.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Search_Weight_Phrase extends Zend_Search_Lucene_Search_Weight
@@ -38,7 +38,7 @@ class Zend_Search_Lucene_Search_Weight_Phrase extends Zend_Search_Lucene_Search_
     /**
      * IndexReader.
      *
-     * @var Zend_Search_Lucene
+     * @var Zend_Search_Lucene_Interface
      */
     private $_reader;
 
@@ -49,13 +49,6 @@ class Zend_Search_Lucene_Search_Weight_Phrase extends Zend_Search_Lucene_Search_
      */
     private $_query;
 
-    /**
-     * Weight value
-     *
-     * @var float
-     */
-    private $_value;
-
     /**
      * Score factor
      *
@@ -63,46 +56,19 @@ class Zend_Search_Lucene_Search_Weight_Phrase extends Zend_Search_Lucene_Search_
      */
     private $_idf;
 
-    /**
-     * Normalization factor
-     *
-     * @var float
-     */
-    private $_queryNorm;
-
-
-    /**
-     * Query weight
-     *
-     * @var float
-     */
-    private $_queryWeight;
-
-
     /**
      * Zend_Search_Lucene_Search_Weight_Phrase constructor
      *
      * @param Zend_Search_Lucene_Search_Query_Phrase $query
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Interface           $reader
      */
-    public function __construct(Zend_Search_Lucene_Search_Query_Phrase $query, Zend_Search_Lucene $reader)
+    public function __construct(Zend_Search_Lucene_Search_Query_Phrase $query,
+                                Zend_Search_Lucene_Interface           $reader)
     {
         $this->_query  = $query;
         $this->_reader = $reader;
     }
 
-
-    /**
-     * The weight for this query
-     *
-     * @return float
-     */
-    public function getValue()
-    {
-        return $this->_value;
-    }
-
-
     /**
      * The sum of squared weights of contained query clauses.
      *

search/Zend/Search/Lucene/Search/Weight/Term.php

@@ -15,20 +15,20 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Search_Weight */
-require_once 'Zend/Search/Lucene/Search/Weight.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Search
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Search_Weight_Term extends Zend_Search_Lucene_Search_Weight
@@ -36,7 +36,7 @@ class Zend_Search_Lucene_Search_Weight_Term extends Zend_Search_Lucene_Search_We
     /**
      * IndexReader.
      *
-     * @var Zend_Search_Lucene
+     * @var Zend_Search_Lucene_Interface
      */
     private $_reader;
 
@@ -54,13 +54,6 @@ class Zend_Search_Lucene_Search_Weight_Term extends Zend_Search_Lucene_Search_We
      */
     private $_query;
 
-    /**
-     * Weight value
-     *
-     * @var float
-     */
-    private $_value;
-
     /**
      * Score factor
      *
@@ -68,14 +61,6 @@ class Zend_Search_Lucene_Search_Weight_Term extends Zend_Search_Lucene_Search_We
      */
     private $_idf;
 
-    /**
-     * Normalization factor
-     *
-     * @var float
-     */
-    private $_queryNorm;
-
-
     /**
      * Query weight
      *
@@ -88,9 +73,13 @@ class Zend_Search_Lucene_Search_Weight_Term extends Zend_Search_Lucene_Search_We
      * Zend_Search_Lucene_Search_Weight_Term constructor
      * reader - index reader
      *
-     * @param Zend_Search_Lucene $reader
+     * @param Zend_Search_Lucene_Index_Term   $term
+     * @param Zend_Search_Lucene_Search_Query $query
+     * @param Zend_Search_Lucene_Interface    $reader
      */
-    public function __construct($term, $query, $reader)
+    public function __construct(Zend_Search_Lucene_Index_Term   $term,
+                                Zend_Search_Lucene_Search_Query $query,
+                                Zend_Search_Lucene_Interface    $reader)
     {
         $this->_term   = $term;
         $this->_query  = $query;
@@ -98,17 +87,6 @@ class Zend_Search_Lucene_Search_Weight_Term extends Zend_Search_Lucene_Search_We
     }
 
 
-    /**
-     * The weight for this query
-     *
-     * @return float
-     */
-    public function getValue()
-    {
-        return $this->_value;
-    }
-
-
     /**
      * The sum of squared weights of contained query clauses.
      *

search/Zend/Search/Lucene/Storage/Directory.php

@@ -15,7 +15,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Storage
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
@@ -24,7 +24,7 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Storage
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 abstract class Zend_Search_Lucene_Storage_Directory
@@ -111,10 +111,16 @@ abstract class Zend_Search_Lucene_Storage_Directory
     /**
      * Returns a Zend_Search_Lucene_Storage_File object for a given $filename in the directory.
      *
+     * If $shareHandler option is true, then file handler can be shared between File Object
+     * requests. It speed-ups performance, but makes problems with file position.
+     * Shared handler are good for short atomic requests.
+     * Non-shared handlers are useful for stream file reading (especial for compound files).
+     *
      * @param string $filename
+     * @param boolean $shareHandler
      * @return Zend_Search_Lucene_Storage_File
      */
-    abstract public function getFileObject($filename);
+    abstract public function getFileObject($filename, $shareHandler = true);
 
 }
 

search/Zend/Search/Lucene/Storage/Directory/Filesystem.php

@@ -15,16 +15,16 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Storage
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Storage_Directory */
-require_once 'Zend/Search/Lucene/Storage/Directory.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/Directory.php';
 
 /** Zend_Search_Lucene_Storage_File_Filesystem */
-require_once 'Zend/Search/Lucene/Storage/File/Filesystem.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/File/Filesystem.php';
 
 
 /**
@@ -33,7 +33,7 @@ require_once 'Zend/Search/Lucene/Storage/File/Filesystem.php';
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Storage
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene_Storage_Directory
@@ -64,7 +64,7 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene
      * @return boolean
      */
 
-    static public function mkdirs($dir, $mode = 0777, $recursive = true)
+    public static function mkdirs($dir, $mode = 0777, $recursive = true)
     {
         if (is_null($dir) || $dir === '') {
             return false;
@@ -113,7 +113,7 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene
             $fileObject->close();
         }
 
-        unset($this->_fileHandlers);
+        $this->_fileHandlers = array();
     }
 
 
@@ -127,15 +127,14 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene
         $result = array();
 
         $dirContent = opendir( $this->_dirPath );
-        while ($file = readdir($dirContent)) {
+        while (($file = readdir($dirContent)) !== false) {
             if (($file == '..')||($file == '.'))   continue;
 
-            $fullName = $this->_dirPath . '/' . $file;
-
             if( !is_dir($this->_dirPath . '/' . $file) ) {
                 $result[] = $file;
             }
         }
+        closedir($dirContent);
 
         return $result;
     }
@@ -165,11 +164,17 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene
      */
     public function deleteFile($filename)
     {
+        /**
+         * @todo add support of "deletable" file
+         * "deletable" is used on Windows systems if file can't be deleted
+         * (while it is still open).
+         */
+
         if (isset($this->_fileHandlers[$filename])) {
             $this->_fileHandlers[$filename]->close();
         }
         unset($this->_fileHandlers[$filename]);
-        unlink($this->_dirPath .'/'. $filename);
+        unlink($this->_dirPath . '/' . $filename);
     }
 
 
@@ -219,24 +224,40 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene
      * @param string $from
      * @param string $to
      * @return void
+     * @throws Zend_Search_Lucene_Exception
      */
     public function renameFile($from, $to)
     {
-        if ($this->_fileHandlers[$from] !== null) {
+        global $php_errormsg;
+
+        if (isset($this->_fileHandlers[$from])) {
             $this->_fileHandlers[$from]->close();
         }
         unset($this->_fileHandlers[$from]);
 
-        if ($this->_fileHandlers[$to] !== null) {
+        if (isset($this->_fileHandlers[$to])) {
             $this->_fileHandlers[$to]->close();
         }
         unset($this->_fileHandlers[$to]);
 
         if (file_exists($this->_dirPath . '/' . $to)) {
-            unlink($this->_dirPath . '/' . $to);
+            if (!unlink($this->_dirPath . '/' . $to)) {
+                throw new Zend_Search_Lucene_Exception('Delete operation failed');
+            }
         }
 
-        return @rename($this->_dirPath . '/' . $from, $this->_dirPath . '/' . $to);
+        $trackErrors = ini_get('track_errors');
+        ini_set('track_errors', '1');
+
+        $success = @rename($this->_dirPath . '/' . $from, $this->_dirPath . '/' . $to);
+        if (!$success) {
+            ini_set('track_errors', $trackErrors);
+            throw new Zend_Search_Lucene_Exception($php_errormsg);
+        }
+
+        ini_set('track_errors', $trackErrors);
+
+        return $success;
     }
 
 
@@ -255,17 +276,29 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene
     /**
      * Returns a Zend_Search_Lucene_Storage_File object for a given $filename in the directory.
      *
+     * If $shareHandler option is true, then file handler can be shared between File Object
+     * requests. It speed-ups performance, but makes problems with file position.
+     * Shared handler are good for short atomic requests.
+     * Non-shared handlers are useful for stream file reading (especial for compound files).
+     *
      * @param string $filename
+     * @param boolean $shareHandler
      * @return Zend_Search_Lucene_Storage_File
      */
-    public function getFileObject($filename)
+    public function getFileObject($filename, $shareHandler = true)
     {
+        $fullFilename = $this->_dirPath . '/' . $filename;
+
+        if (!$shareHandler) {
+            return new Zend_Search_Lucene_Storage_File_Filesystem($fullFilename);
+        }
+
         if (isset( $this->_fileHandlers[$filename] )) {
             $this->_fileHandlers[$filename]->seek(0);
             return $this->_fileHandlers[$filename];
         }
 
-        $this->_fileHandlers[$filename] = new Zend_Search_Lucene_Storage_File_Filesystem($this->_dirPath . '/' . $filename);
+        $this->_fileHandlers[$filename] = new Zend_Search_Lucene_Storage_File_Filesystem($fullFilename);
         return $this->_fileHandlers[$filename];
     }
 }

search/Zend/Search/Lucene/Storage/File.php

@@ -15,21 +15,21 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Storage
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 
 /** Zend_Search_Lucene_Exception */
-require_once 'Zend/Search/Lucene/Exception.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Storage
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 abstract class Zend_Search_Lucene_Storage_File
@@ -69,6 +69,15 @@ abstract class Zend_Search_Lucene_Storage_File
      */
     abstract public function tell();
 
+    /**
+     * Flush output.
+     *
+     * Returns true on success or false on failure.
+     *
+     * @return boolean
+     */
+    abstract public function flush();
+
     /**
      * Writes $length number of bytes (all, if $length===null) to the end
      * of the file.
@@ -78,6 +87,20 @@ abstract class Zend_Search_Lucene_Storage_File
      */
     abstract protected function _fwrite($data, $length=null);
 
+    /**
+     * Lock file
+     *
+     * Lock type may be a LOCK_SH (shared lock) or a LOCK_EX (exclusive lock)
+     *
+     * @param integer $lockType
+     * @return boolean
+     */
+    abstract public function lock($lockType, $nonBlockinLock = false);
+
+    /**
+     * Unlock file
+     */
+    abstract public function unlock();
 
     /**
      * Reads a byte from the current position in the file
@@ -401,4 +424,4 @@ abstract class Zend_Search_Lucene_Storage_File
     {
         return $this->_fread($this->readVInt());
     }
-}
+}

search/Zend/Search/Lucene/Storage/File/Filesystem.php

@@ -15,23 +15,23 @@
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Storage
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 
 
 /** Zend_Search_Lucene_Storage_File */
-require_once 'Zend/Search/Lucene/Storage/File.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/File.php';
 
 /** Zend_Search_Lucene_Exception */
-require_once 'Zend/Search/Lucene/Exception.php';
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
 
 
 /**
  * @category   Zend
  * @package    Zend_Search_Lucene
  * @subpackage Storage
- * @copyright  Copyright (c) 2006 Zend Technologies USA Inc. (http://www.zend.com)
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  */
 class Zend_Search_Lucene_Storage_File_Filesystem extends Zend_Search_Lucene_Storage_File
@@ -53,12 +53,12 @@ class Zend_Search_Lucene_Storage_File_Filesystem extends Zend_Search_Lucene_Stor
     {
         global $php_errormsg;
 
-        $trackErrors = ini_get( "track_errors");
+        $trackErrors = ini_get('track_errors');
         ini_set('track_errors', '1');
 
         $this->_fileHandle = @fopen($filename, $mode);
 
-        if ($this->_fileHandle===false) {
+        if ($this->_fileHandle === false) {
             ini_set('track_errors', $trackErrors);
             throw new Zend_Search_Lucene_Exception($php_errormsg);
         }
@@ -100,6 +100,17 @@ class Zend_Search_Lucene_Storage_File_Filesystem extends Zend_Search_Lucene_Stor
         return ftell($this->_fileHandle);
     }
 
+    /**
+     * Flush output.
+     *
+     * Returns true on success or false on failure.
+     *
+     * @return boolean
+     */
+    public function flush()
+    {
+        return fflush($this->_fileHandle);
+    }
 
     /**
      * Close File object
@@ -167,5 +178,39 @@ class Zend_Search_Lucene_Storage_File_Filesystem extends Zend_Search_Lucene_Stor
             fwrite($this->_fileHandle, $data, $length);
         }
     }
+
+    /**
+     * Lock file
+     *
+     * Lock type may be a LOCK_SH (shared lock) or a LOCK_EX (exclusive lock)
+     *
+     * @param integer $lockType
+     * @param boolean $nonBlockinLock
+     * @return boolean
+     */
+    public function lock($lockType, $nonBlockinLock = false)
+    {
+        if ($nonBlockinLock) {
+            return flock($this->_fileHandle, $lockType | LOCK_NB);
+        } else {
+            return flock($this->_fileHandle, $lockType);
+        }
+    }
+
+    /**
+     * Unlock file
+     *
+     * Returns true on success
+     *
+     * @return boolean
+     */
+    public function unlock()
+    {
+        if ($this->_fileHandle !== null ) {
+            return flock($this->_fileHandle, LOCK_UN);
+        } else {
+            return true;
+        }
+    }
 }
 

search/Zend/Search/Lucene/Storage/File/Memory.php

@@ -0,0 +1,555 @@
+<?php
+/**
+ * Zend Framework
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * It is also available through the world-wide-web at this URL:
+ * http://framework.zend.com/license/new-bsd
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Storage
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+
+
+/** Zend_Search_Lucene_Storage_File */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/File.php';
+
+/** Zend_Search_Lucene_Exception */
+require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php';
+
+
+/**
+ * @category   Zend
+ * @package    Zend_Search_Lucene
+ * @subpackage Storage
+ * @copyright  Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com)
+ * @license    http://framework.zend.com/license/new-bsd     New BSD License
+ */
+class Zend_Search_Lucene_Storage_File_Memory extends Zend_Search_Lucene_Storage_File
+{
+    /**
+     * FileData
+     *
+     * @var string
+     */
+    private $_data;
+
+    /**
+     * File Position
+     *
+     * @var integer
+     */
+    private $_position = 0;
+
+
+    /**
+     * Object constractor
+     *
+     * @param string $data
+     */
+    public function __construct($data)
+    {
+        $this->_data = $data;
+    }
+
+    /**
+     * Reads $length number of bytes at the current position in the
+     * file and advances the file pointer.
+     *
+     * @param integer $length
+     * @return string
+     */
+    protected function _fread($length = 1)
+    {
+        $returnValue = substr($this->_data, $this->_position, $length);
+        $this->_position += $length;
+        return $returnValue;
+    }
+
+
+    /**
+     * Sets the file position indicator and advances the file pointer.
+     * The new position, measured in bytes from the beginning of the file,
+     * is obtained by adding offset to the position specified by whence,
+     * whose values are defined as follows:
+     * SEEK_SET - Set position equal to offset bytes.
+     * SEEK_CUR - Set position to current location plus offset.
+     * SEEK_END - Set position to end-of-file plus offset. (To move to
+     * a position before the end-of-file, you need to pass a negative value
+     * in offset.)
+     * Upon success, returns 0; otherwise, returns -1
+     *
+     * @param integer $offset
+     * @param integer $whence
+     * @return integer
+     */
+    public function seek($offset, $whence=SEEK_SET)
+    {
+        switch ($whence) {
+            case SEEK_SET:
+                $this->_position = $offset;
+                break;
+
+            case SEEK_CUR:
+                $this->_position += $offset;
+                break;
+
+            case SEEK_END:
+                $this->_position = strlen($this->_data);
+                $this->_position += $offset;
+                break;
+
+            default:
+                break;
+        }
+    }
+
+    /**
+     * Get file position.
+     *
+     * @return integer
+     */
+    public function tell()
+    {
+        return $this->_position;
+    }
+
+    /**
+     * Flush output.
+     *
+     * Returns true on success or false on failure.
+     *
+     * @return boolean
+     */
+    public function flush()
+    {
+        // Do nothing
+
+        return true;
+    }
+
+    /**
+     * Writes $length number of bytes (all, if $length===null) to the end
+     * of the file.
+     *
+     * @param string $data
+     * @param integer $length
+     */
+    protected function _fwrite($data, $length=null)
+    {
+        // We do not need to check if file position points to the end of "file".
+        // Only append operation is supported now
+
+        if ($length !== null) {
+            $this->_data .= substr($data, 0, $length);
+        } else {
+            $this->_data .= $data;
+        }
+
+        $this->_position = strlen($this->_data);
+    }
+
+    /**
+     * Lock file
+     *
+     * Lock type may be a LOCK_SH (shared lock) or a LOCK_EX (exclusive lock)
+     *
+     * @param integer $lockType
+     * @return boolean
+     */
+    public function lock($lockType, $nonBlockinLock = false)
+    {
+        // Memory files can't be shared
+        // do nothing
+
+        return true;
+    }
+
+    /**
+     * Unlock file
+     */
+    public function unlock()
+    {
+        // Memory files can't be shared
+        // do nothing
+    }
+
+    /**
+     * Reads a byte from the current position in the file
+     * and advances the file pointer.
+     *
+     * @return integer
+     */
+    public function readByte()
+    {
+        return ord($this->_data[$this->_position++]);
+    }
+
+    /**
+     * Writes a byte to the end of the file.
+     *
+     * @param integer $byte
+     */
+    public function writeByte($byte)
+    {
+        // We do not need to check if file position points to the end of "file".
+        // Only append operation is supported now
+
+        $this->_data .= chr($byte);
+        $this->_position = strlen($this->_data);
+
+        return 1;
+    }
+
+    /**
+     * Read num bytes from the current position in the file
+     * and advances the file pointer.
+     *
+     * @param integer $num
+     * @return string
+     */
+    public function readBytes($num)
+    {
+        $returnValue = substr($this->_data, $this->_position, $num);
+        $this->_position += $num;
+
+        return $returnValue;
+    }
+
+    /**
+     * Writes num bytes of data (all, if $num===null) to the end
+     * of the string.
+     *
+     * @param string $data
+     * @param integer $num
+     */
+    public function writeBytes($data, $num=null)
+    {
+        // We do not need to check if file position points to the end of "file".
+        // Only append operation is supported now
+
+        if ($num !== null) {
+            $this->_data .= substr($data, 0, $num);
+        } else {
+            $this->_data .= $data;
+        }
+
+        $this->_position = strlen($this->_data);
+    }
+
+
+    /**
+     * Reads an integer from the current position in the file
+     * and advances the file pointer.
+     *
+     * @return integer
+     */
+    public function readInt()
+    {
+        $str = substr($this->_data, $this->_position, 4);
+        $this->_position += 4;
+
+        return  ord($str{0}) << 24 |
+                ord($str{1}) << 16 |
+                ord($str{2}) << 8  |
+                ord($str{3});
+    }
+
+
+    /**
+     * Writes an integer to the end of file.
+     *
+     * @param integer $value
+     */
+    public function writeInt($value)
+    {
+        // We do not need to check if file position points to the end of "file".
+        // Only append operation is supported now
+
+        settype($value, 'integer');
+        $this->_data .= chr($value>>24 & 0xFF) .
+                        chr($value>>16 & 0xFF) .
+                        chr($value>>8  & 0xFF) .
+                        chr($value     & 0xFF);
+
+        $this->_position = strlen($this->_data);
+    }
+
+
+    /**
+     * Returns a long integer from the current position in the file
+     * and advances the file pointer.
+     *
+     * @return integer
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public function readLong()
+    {
+        $str = substr($this->_data, $this->_position, 8);
+        $this->_position += 8;
+
+        /**
+         * Check, that we work in 64-bit mode.
+         * fseek() uses long for offset. Thus, largest index segment file size in 32bit mode is 2Gb
+         */
+        if (PHP_INT_SIZE > 4) {
+            return  ord($str{0}) << 56  |
+                    ord($str{1}) << 48  |
+                    ord($str{2}) << 40  |
+                    ord($str{3}) << 32  |
+                    ord($str{4}) << 24  |
+                    ord($str{5}) << 16  |
+                    ord($str{6}) << 8   |
+                    ord($str{7});
+        } else {
+            if ((ord($str{0})          != 0) ||
+                (ord($str{1})          != 0) ||
+                (ord($str{2})          != 0) ||
+                (ord($str{3})          != 0) ||
+                ((ord($str{0}) & 0x80) != 0)) {
+                     throw new Zend_Search_Lucene_Exception('Largest supported segment size (for 32-bit mode) is 2Gb');
+                 }
+
+            return  ord($str{4}) << 24  |
+                    ord($str{5}) << 16  |
+                    ord($str{6}) << 8   |
+                    ord($str{7});
+        }
+    }
+
+    /**
+     * Writes long integer to the end of file
+     *
+     * @param integer $value
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public function writeLong($value)
+    {
+        // We do not need to check if file position points to the end of "file".
+        // Only append operation is supported now
+
+        /**
+         * Check, that we work in 64-bit mode.
+         * fseek() and ftell() use long for offset. Thus, largest index segment file size in 32bit mode is 2Gb
+         */
+        if (PHP_INT_SIZE > 4) {
+            settype($value, 'integer');
+            $this->_data .= chr($value>>56 & 0xFF) .
+                            chr($value>>48 & 0xFF) .
+                            chr($value>>40 & 0xFF) .
+                            chr($value>>32 & 0xFF) .
+                            chr($value>>24 & 0xFF) .
+                            chr($value>>16 & 0xFF) .
+                            chr($value>>8  & 0xFF) .
+                            chr($value     & 0xFF);
+        } else {
+            if ($value > 0x7FFFFFFF) {
+                throw new Zend_Search_Lucene_Exception('Largest supported segment size (for 32-bit mode) is 2Gb');
+            }
+
+            $this->_data .= chr(0) . chr(0) . chr(0) . chr(0) .
+                            chr($value>>24 & 0xFF) .
+                            chr($value>>16 & 0xFF) .
+                            chr($value>>8  & 0xFF) .
+                            chr($value     & 0xFF);
+        }
+
+        $this->_position = strlen($this->_data);
+    }
+
+
+
+    /**
+     * Returns a variable-length integer from the current
+     * position in the file and advances the file pointer.
+     *
+     * @return integer
+     */
+    public function readVInt()
+    {
+        $nextByte = ord($this->_data[$this->_position++]);
+        $val = $nextByte & 0x7F;
+
+        for ($shift=7; ($nextByte & 0x80) != 0; $shift += 7) {
+            $nextByte = ord($this->_data[$this->_position++]);
+            $val |= ($nextByte & 0x7F) << $shift;
+        }
+        return $val;
+    }
+
+    /**
+     * Writes a variable-length integer to the end of file.
+     *
+     * @param integer $value
+     */
+    public function writeVInt($value)
+    {
+        // We do not need to check if file position points to the end of "file".
+        // Only append operation is supported now
+
+        settype($value, 'integer');
+        while ($value > 0x7F) {
+            $this->_data .= chr( ($value & 0x7F)|0x80 );
+            $value >>= 7;
+        }
+        $this->_data .= chr($value);
+
+        $this->_position = strlen($this->_data);
+    }
+
+
+    /**
+     * Reads a string from the current position in the file
+     * and advances the file pointer.
+     *
+     * @return string
+     */
+    public function readString()
+    {
+        $strlen = $this->readVInt();
+        if ($strlen == 0) {
+            return '';
+        } else {
+            /**
+             * This implementation supports only Basic Multilingual Plane
+             * (BMP) characters (from 0x0000 to 0xFFFF) and doesn't support
+             * "supplementary characters" (characters whose code points are
+             * greater than 0xFFFF)
+             * Java 2 represents these characters as a pair of char (16-bit)
+             * values, the first from the high-surrogates range (0xD800-0xDBFF),
+             * the second from the low-surrogates range (0xDC00-0xDFFF). Then
+             * they are encoded as usual UTF-8 characters in six bytes.
+             * Standard UTF-8 representation uses four bytes for supplementary
+             * characters.
+             */
+
+            $str_val = substr($this->_data, $this->_position, $strlen);
+            $this->_position += $strlen;
+
+            for ($count = 0; $count < $strlen; $count++ ) {
+                if (( ord($str_val{$count}) & 0xC0 ) == 0xC0) {
+                    $addBytes = 1;
+                    if (ord($str_val{$count}) & 0x20 ) {
+                        $addBytes++;
+
+                        // Never used. Java2 doesn't encode strings in four bytes
+                        if (ord($str_val{$count}) & 0x10 ) {
+                            $addBytes++;
+                        }
+                    }
+                    $str_val .= substr($this->_data, $this->_position, $addBytes);
+                    $this->_position += $addBytes;
+                    $strlen          += $addBytes;
+
+                    // Check for null character. Java2 encodes null character
+                    // in two bytes.
+                    if (ord($str_val{$count})   == 0xC0 &&
+                        ord($str_val{$count+1}) == 0x80   ) {
+                        $str_val{$count} = 0;
+                        $str_val = substr($str_val,0,$count+1)
+                                 . substr($str_val,$count+2);
+                    }
+                    $count += $addBytes;
+                }
+            }
+
+            return $str_val;
+        }
+    }
+
+    /**
+     * Writes a string to the end of file.
+     *
+     * @param string $str
+     * @throws Zend_Search_Lucene_Exception
+     */
+    public function writeString($str)
+    {
+        /**
+         * This implementation supports only Basic Multilingual Plane
+         * (BMP) characters (from 0x0000 to 0xFFFF) and doesn't support
+         * "supplementary characters" (characters whose code points are
+         * greater than 0xFFFF)
+         * Java 2 represents these characters as a pair of char (16-bit)
+         * values, the first from the high-surrogates range (0xD800-0xDBFF),
+         * the second from the low-surrogates range (0xDC00-0xDFFF). Then
+         * they are encoded as usual UTF-8 characters in six bytes.
+         * Standard UTF-8 representation uses four bytes for supplementary
+         * characters.
+         */
+
+        // We do not need to check if file position points to the end of "file".
+        // Only append operation is supported now
+
+        // convert input to a string before iterating string characters
+        settype($str, 'string');
+
+        $chars = $strlen = strlen($str);
+        $containNullChars = false;
+
+        for ($count = 0; $count < $strlen; $count++ ) {
+            /**
+             * String is already in Java 2 representation.
+             * We should only calculate actual string length and replace
+             * \x00 by \xC0\x80
+             */
+            if ((ord($str{$count}) & 0xC0) == 0xC0) {
+                $addBytes = 1;
+                if (ord($str{$count}) & 0x20 ) {
+                    $addBytes++;
+
+                    // Never used. Java2 doesn't encode strings in four bytes
+                    // and we dont't support non-BMP characters
+                    if (ord($str{$count}) & 0x10 ) {
+                        $addBytes++;
+                    }
+                }
+                $chars -= $addBytes;
+
+                if (ord($str{$count}) == 0 ) {
+                    $containNullChars = true;
+                }
+                $count += $addBytes;
+            }
+        }
+
+        if ($chars < 0) {
+            throw new Zend_Search_Lucene_Exception('Invalid UTF-8 string');
+        }
+
+        $this->writeVInt($chars);
+        if ($containNullChars) {
+            $this->_data .= str_replace($str, "\x00", "\xC0\x80");
+
+        } else {
+            $this->_data .= $str;
+        }
+
+        $this->_position = strlen($this->_data);
+    }
+
+
+    /**
+     * Reads binary data from the current position in the file
+     * and advances the file pointer.
+     *
+     * @return string
+     */
+    public function readBinary()
+    {
+        $length = $this->readVInt();
+        $returnValue = substr($this->_data, $this->_position, $length);
+        $this->_position += $length;
+        return $returnValue;
+    }
+}
+

search/Zend/Search/TODO.txt

@@ -1,14 +1,7 @@
 @todo
 
-- Improve API: fix ZSearchMultiTermQuery($terms, $signs);
-
-- Analysis and indexing engine
-
-- Additional queries: phrase, wildcard, proximity, and range
+- Additional queries: wildcard, proximity, and range
 
 - Better class-level docblocks (most functions okay)
 
-- Some Windows issues(?) during indexing
-
-- Finish renaming classes to PEAR-like conventions