Sync with latest clucene source. Fixes at least two crash bugs.

6 files changed, 111 insertions, 66 deletions

diff --git a/src/3rdparty/clucene/src/CLucene/index/SegmentTermDocs.cpp b/src/3rdparty/clucene/src/CLucene/index/SegmentTermDocs.cpp
index f4c5e3a..50951e9 100644
--- a/src/3rdparty/clucene/src/CLucene/index/SegmentTermDocs.cpp
+++ b/src/3rdparty/clucene/src/CLucene/index/SegmentTermDocs.cpp
@@ -112,47 +112,51 @@ CL_NS_DEF(index)
 	  return _freq; 
   }
 
-  bool SegmentTermDocs::next() {
-    while (true) {
-      if (count == df)
-        return false;
 
-      uint32_t docCode = freqStream->readVInt();
-      _doc += docCode >> 1; //unsigned shift
-      if ((docCode & 1) != 0)			  // if low bit is set
-        _freq = 1;				  // _freq is one
-      else
-        _freq = freqStream->readVInt();		  // else read _freq
-      count++;
-
-      if ( (deletedDocs == NULL) || (deletedDocs->get(_doc) == false ) )
-        break;
-      skippingDoc();
+bool SegmentTermDocs::next()
+{
+    while (true) {
+        if (count == df)
+            return false;
+
+        uint32_t docCode = freqStream->readVInt();
+        _doc += docCode >> 1;   //unsigned shift
+        if ((docCode & 1) != 0)             // if low bit is set
+            _freq = 1;                      // _freq is one
+        else
+            _freq = freqStream->readVInt(); // else read _freq
+        count++;
+
+        if (deletedDocs == NULL || (_doc >= 0 && !deletedDocs->get(_doc)))
+            break;
+        skippingDoc();
     }
     return true;
-  }
+}
 
-  int32_t SegmentTermDocs::read(int32_t* docs, int32_t* freqs, int32_t length) {
+
+int32_t SegmentTermDocs::read(int32_t* docs, int32_t* freqs, int32_t length)
+{
     int32_t i = 0;
-//todo: one optimization would be to get the pointer buffer for ram or mmap dirs 
-//and iterate over them instead of using readByte() intensive functions.
-    while (i<length && count < df) {
-      uint32_t docCode = freqStream->readVInt();
-      _doc += docCode >> 1;
-      if ((docCode & 1) != 0)			  // if low bit is set
-        _freq = 1;				  // _freq is one
-      else
-        _freq = freqStream->readVInt();		  // else read _freq
-      count++;
-
-      if (deletedDocs == NULL || !deletedDocs->get(_doc)) {
-        docs[i] = _doc;
-        freqs[i] = _freq;
-        i++;
-      }
+    // TODO: one optimization would be to get the pointer buffer for ram or mmap
+    // dirs and iterate over them instead of using readByte() intensive functions.
+    while (i < length && count < df) {
+        uint32_t docCode = freqStream->readVInt();
+        _doc += docCode >> 1;
+        if ((docCode & 1) != 0)             // if low bit is set
+            _freq = 1;                      // _freq is one
+        else
+            _freq = freqStream->readVInt(); // else read _freq
+        count++;
+
+        if (deletedDocs == NULL || (_doc >= 0 && !deletedDocs->get(_doc))) {
+            docs[i] = _doc;
+            freqs[i] = _freq;
+            i++;
+        }
     }
     return i;
-  }
+}
 
   bool SegmentTermDocs::skipTo(const int32_t target){
     if (df >= skipInterval) {                      // optimized case
diff --git a/src/3rdparty/clucene/src/CLucene/index/Term.cpp b/src/3rdparty/clucene/src/CLucene/index/Term.cpp
index fc32e44..5ff7bb2 100644
--- a/src/3rdparty/clucene/src/CLucene/index/Term.cpp
+++ b/src/3rdparty/clucene/src/CLucene/index/Term.cpp
@@ -153,7 +153,10 @@ int32_t Term::compareTo(const Term* other) const
     if (_field == other->_field)
         return _tcscmp(_text, other->_text);
 
-    return _tcscmp(_field, other->_field);
+    int32_t ret = _tcscmp(_field, other->_field);
+    if (ret == 0)
+        ret = _tcscmp(_text, other->_text);
+    return ret;
 }
 
 TCHAR* Term::toString() const
diff --git a/src/3rdparty/clucene/src/CLucene/queryParser/MultiFieldQueryParser.cpp b/src/3rdparty/clucene/src/CLucene/queryParser/MultiFieldQueryParser.cpp
index ea93ec4..b57896b 100644
--- a/src/3rdparty/clucene/src/CLucene/queryParser/MultiFieldQueryParser.cpp
+++ b/src/3rdparty/clucene/src/CLucene/queryParser/MultiFieldQueryParser.cpp
@@ -21,51 +21,62 @@ CL_NS_USE(analysis)
 
 CL_NS_DEF(queryParser)
 
-MultiFieldQueryParser::MultiFieldQueryParser(const TCHAR** fields, CL_NS(analysis)::Analyzer* a, BoostMap* boosts):
-	QueryParser(NULL,a)
+MultiFieldQueryParser::MultiFieldQueryParser(const TCHAR** fields,
+    CL_NS(analysis)::Analyzer* analyzer, BoostMap* boosts)
+    : QueryParser(NULL, analyzer)
 {
 	this->fields = fields;
     this->boosts = boosts;
 }
-MultiFieldQueryParser::~MultiFieldQueryParser(){
+
+MultiFieldQueryParser::~MultiFieldQueryParser()
+{
 }
 
 //static 
-Query* MultiFieldQueryParser::parse(const TCHAR* query, const TCHAR** fields, Analyzer* analyzer)
+Query* MultiFieldQueryParser::parse(const TCHAR* query, const TCHAR** fields,
+    Analyzer* analyzer)
 {
     BooleanQuery* bQuery = _CLNEW BooleanQuery();
     int32_t i = 0;
-    while ( fields[i] != NULL ){
-		   Query* q = QueryParser::parse(query, fields[i], analyzer);
-			bQuery->add(q, true, false, false);
-
+    while (fields[i] != NULL){
+        Query* q = QueryParser::parse(query, fields[i], analyzer);
+        if (q && (q->getQueryName() != _T("BooleanQuery")
+          || ((BooleanQuery*)q)->getClauseCount() > 0)) {
+            bQuery->add(q , true, false, false);
+        } else {
+            _CLDELETE(q);
+        }
         i++;
     }
     return bQuery;
 }
 
 //static 
-Query* MultiFieldQueryParser::parse(const TCHAR* query, const TCHAR** fields, const uint8_t* flags, Analyzer* analyzer)
+Query* MultiFieldQueryParser::parse(const TCHAR* query, const TCHAR** fields,
+    const uint8_t* flags, Analyzer* analyzer)
 {
     BooleanQuery* bQuery = _CLNEW BooleanQuery();
     int32_t i = 0;
-    while ( fields[i] != NULL )
-    {
-		Query* q = QueryParser::parse(query, fields[i], analyzer);
-        uint8_t flag = flags[i];
-        switch (flag)
-        {
-			case MultiFieldQueryParser::REQUIRED_FIELD:
-                bQuery->add(q, true, true, false);
-                break;
-            case MultiFieldQueryParser::PROHIBITED_FIELD:
-                bQuery->add(q, true, false, true);
-                break;
-            default:
-                bQuery->add(q, true, false, false);
-                break;
+    while ( fields[i] != NULL ) {
+        Query* q = QueryParser::parse(query, fields[i], analyzer);
+        if (q && (q->getQueryName() != _T("BooleanQuery")
+          || ((BooleanQuery*)q)->getClauseCount() > 0)) {
+            uint8_t flag = flags[i];
+            switch (flag) {
+                case MultiFieldQueryParser::REQUIRED_FIELD:
+                    bQuery->add(q, true, true, false);
+                    break;
+                case MultiFieldQueryParser::PROHIBITED_FIELD:
+                    bQuery->add(q, true, false, true);
+                    break;
+                default:
+                    bQuery->add(q, true, false, false);
+                    break;
+            }
+        } else {
+            _CLDELETE(q);
         }
-
         i++;
     }
     return bQuery;
diff --git a/src/3rdparty/clucene/src/CLucene/store/FSDirectory.cpp b/src/3rdparty/clucene/src/CLucene/store/FSDirectory.cpp
index e9659cf..5f96e91 100644
--- a/src/3rdparty/clucene/src/CLucene/store/FSDirectory.cpp
+++ b/src/3rdparty/clucene/src/CLucene/store/FSDirectory.cpp
@@ -91,7 +91,7 @@ QString FSDirectory::FSLock::toString() const
 // # pragma mark -- FSDirectory::FSIndexInput
 
 FSDirectory::FSIndexInput::FSIndexInput(const QString& path, int32_t bufferSize)
-    : BufferedIndexInput(bufferSize)	
+    : BufferedIndexInput(bufferSize)
 {
     CND_PRECONDITION(!path.isEmpty(), "path is NULL");
 
@@ -155,7 +155,7 @@ FSDirectory::FSIndexInput::FSIndexInput(const FSIndexInput& other)
     if (other.handle == NULL)
         _CLTHROWA(CL_ERR_NullPointer, "other handle is null");
 
-    SCOPED_LOCK_MUTEX(other.handle->THIS_LOCK)
+    SCOPED_LOCK_MUTEX(*other.handle->THIS_LOCK)
 
     _pos = other.handle->_fpos;
     handle = _CL_POINTER(other.handle);
@@ -169,7 +169,30 @@ FSDirectory::FSIndexInput::~FSIndexInput()
 void FSDirectory::FSIndexInput::close()
 {
     BufferedIndexInput::close();
+#ifdef _LUCENE_THREADMUTEX
+    if (handle != NULL) {
+        // Here we have a bit of a problem... We need to lock the handle to
+        // ensure that we can safely delete the handle... But if we delete the
+        // handle, then the scoped unlock, won't be able to unlock the mutex...
+
+        // take a reference of the lock object...
+        _LUCENE_THREADMUTEX* mutex = handle->THIS_LOCK;
+        //lock the mutex
+        mutex->lock();
+
+        // determine if we are about to delete the handle...
+        bool doUnlock = (handle->__cl_refcount > 1);
+        // decdelete (deletes if refcount is down to 0)
+        _CLDECDELETE(handle);
+
+        if (doUnlock)
+            mutex->unlock();
+        else
+            delete mutex;
+    }
+#else
     _CLDECDELETE(handle);
+#endif
 }
 
 IndexInput* FSDirectory::FSIndexInput::clone() const
@@ -186,7 +209,7 @@ void FSDirectory::FSIndexInput::seekInternal(const int64_t position)
 
 void FSDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_t len)
 {
-    SCOPED_LOCK_MUTEX(handle->THIS_LOCK)
+    SCOPED_LOCK_MUTEX(*handle->THIS_LOCK)
 
     CND_PRECONDITION(handle != NULL, "shared file handle has closed");
     CND_PRECONDITION(handle->fhandle.isOpen(), "file is not open");
@@ -214,8 +237,10 @@ void FSDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_t len)
 FSDirectory::FSIndexInput::SharedHandle::SharedHandle()
     : _fpos(0)
     , _length(0)
-
 {
+#ifdef _LUCENE_THREADMUTEX
+    THIS_LOCK = new _LUCENE_THREADMUTEX;
+#endif
 }
 
 FSDirectory::FSIndexInput::SharedHandle::~SharedHandle()
diff --git a/src/3rdparty/clucene/src/CLucene/store/FSDirectory.h b/src/3rdparty/clucene/src/CLucene/store/FSDirectory.h
index 1302edf..e967380 100644
--- a/src/3rdparty/clucene/src/CLucene/store/FSDirectory.h
+++ b/src/3rdparty/clucene/src/CLucene/store/FSDirectory.h
@@ -147,7 +147,7 @@ private:
 
         void close();
         IndexInput* clone() const;
-                
+
         int64_t length()
         { return handle->_length; }
 
@@ -174,7 +174,7 @@ private:
             int64_t _length;
             
             QFile fhandle;
-            DEFINE_MUTEX(THIS_LOCK)
+            DEFINE_MUTEX(*THIS_LOCK)
         };
         SharedHandle* handle;
         int64_t _pos;
diff --git a/src/3rdparty/clucene/src/CLucene/util/bufferedstream.h b/src/3rdparty/clucene/src/CLucene/util/bufferedstream.h
index 2455d5e..b73ad98 100644
--- a/src/3rdparty/clucene/src/CLucene/util/bufferedstream.h
+++ b/src/3rdparty/clucene/src/CLucene/util/bufferedstream.h
@@ -28,7 +28,9 @@
 
 #include "streambase.h"
 #include "inputstreambuffer.h"
+
 #include <cassert>
+#include <stdio.h>
 
 namespace jstreams {