Long live Qt 4.5!

1 files changed, 481 insertions, 0 deletions

diff --git a/tools/assistant/lib/qhelpsearchindexwriter_clucene.cpp b/tools/assistant/lib/qhelpsearchindexwriter_clucene.cpp
new file mode 100644
index 0000000..e53bbae
--- /dev/null
+++ b/tools/assistant/lib/qhelpsearchindexwriter_clucene.cpp
@@ -0,0 +1,481 @@
+/****************************************************************************
+**
+** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: Qt Software Information (qt-info@nokia.com)
+**
+** This file is part of the Qt Assistant of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the either Technology Preview License Agreement or the
+** Beta Release License Agreement.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain
+** additional rights. These rights are described in the Nokia Qt LGPL
+** Exception version 1.0, included in the file LGPL_EXCEPTION.txt in this
+** package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3.0 as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU General Public License version 3.0 requirements will be
+** met: http://www.gnu.org/copyleft/gpl.html.
+**
+** If you are unsure which license is appropriate for your use, please
+** contact the sales department at qt-sales@nokia.com.
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qhelpenginecore.h"
+#include "qhelp_global.h"
+#include "fulltextsearch/qhits_p.h"
+#include "fulltextsearch/qquery_p.h"
+#include "fulltextsearch/qanalyzer_p.h"
+#include "fulltextsearch/qdocument_p.h"
+#include "fulltextsearch/qsearchable_p.h"
+#include "fulltextsearch/qindexreader_p.h"
+#include "fulltextsearch/qindexwriter_p.h"
+#include "qhelpsearchindexwriter_clucene_p.h"
+
+#include <QtCore/QDir>
+#include <QtCore/QString>
+#include <QtCore/QFileInfo>
+#include <QtCore/QTextCodec>
+#include <QtCore/QTextStream>
+
+#include <QtNetwork/QLocalSocket>
+#include <QtNetwork/QLocalServer>
+
+QT_BEGIN_NAMESPACE
+
+namespace qt {
+    namespace fulltextsearch {
+        namespace clucene {
+
+class DocumentHelper
+{
+public:
+    DocumentHelper(const QString& fileName, const QByteArray &data)
+        : fileName(fileName) , data(readData(data)) {}
+    ~DocumentHelper() {}
+
+    bool addFieldsToDocument(QCLuceneDocument *document,
+        const QString &namespaceName, const QString &attributes = QString())
+    {
+        if (!document)
+            return false;
+
+        if(!data.isEmpty()) {
+            QString parsedData = parseData();
+            QString parsedTitle = QHelpGlobal::documentTitle(data);
+
+            if(!parsedData.isEmpty()) {
+                document->add(new QCLuceneField(QLatin1String("content"),
+                    parsedData,QCLuceneField::INDEX_TOKENIZED));
+                document->add(new QCLuceneField(QLatin1String("path"), fileName,
+                    QCLuceneField::STORE_YES | QCLuceneField::INDEX_UNTOKENIZED));
+                document->add(new QCLuceneField(QLatin1String("title"), parsedTitle,
+                    QCLuceneField::STORE_YES | QCLuceneField::INDEX_UNTOKENIZED));
+                document->add(new QCLuceneField(QLatin1String("titleTokenized"), parsedTitle,
+                    QCLuceneField::STORE_YES | QCLuceneField::INDEX_TOKENIZED));
+                document->add(new QCLuceneField(QLatin1String("namespace"), namespaceName,
+                    QCLuceneField::STORE_YES | QCLuceneField::INDEX_UNTOKENIZED));
+                document->add(new QCLuceneField(QLatin1String("attribute"), attributes,
+                    QCLuceneField::STORE_YES | QCLuceneField::INDEX_TOKENIZED));
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+private:
+    QString readData(const QByteArray &data)
+    {
+        QTextStream textStream(data);
+        QByteArray charSet = QHelpGlobal::charsetFromData(data).toLatin1();
+        textStream.setCodec(QTextCodec::codecForName(charSet.constData()));
+
+        QString stream = textStream.readAll();
+        if (stream.isNull() || stream.isEmpty())
+            return QString();
+
+        return stream;
+    }
+
+    QString parseData() const
+    {
+        const int length = data.length();
+        const QChar *buf = data.unicode();
+
+        QString parsedContent;
+        parsedContent.reserve(length);
+
+        bool valid = true;
+        int j = 0, count = 0;
+
+        QChar c;
+        while (j < length) {
+            c = buf[j++];
+            if (c == QLatin1Char('<') || c == QLatin1Char('&')) {
+                if (count > 1)
+                    parsedContent.append(QLatin1Char(' '));
+                count = 0;
+                valid = false;
+                continue;
+            }
+            if ((c == QLatin1Char('>') || c == QLatin1Char(';')) && !valid) {
+                valid = true;
+                continue;
+            }
+            if (!valid)
+                continue;
+
+            if (c.isLetterOrNumber() || c.isPrint()) {
+                ++count;
+                parsedContent.append(c.toLower());
+            } else {
+                if (count > 1)
+                    parsedContent.append(QLatin1Char(' '));
+                count = 0;
+            }
+        }
+
+        return parsedContent;
+    }
+
+private:
+    QString fileName;
+    QString data;
+};
+
+
+QHelpSearchIndexWriter::QHelpSearchIndexWriter()
+    : QThread(0)
+    , m_cancel(false)
+{
+    // nothing todo
+}
+
+QHelpSearchIndexWriter::~QHelpSearchIndexWriter()
+{
+    mutex.lock();
+    this->m_cancel = true;
+    waitCondition.wakeOne();
+    mutex.unlock();
+
+    wait();
+}
+
+void QHelpSearchIndexWriter::cancelIndexing()
+{
+    mutex.lock();
+    this->m_cancel = true;
+    mutex.unlock();
+}
+
+void QHelpSearchIndexWriter::updateIndex(const QString &collectionFile,
+    const QString &indexFilesFolder, bool reindex)
+{
+    mutex.lock();
+    this->m_cancel = false;
+    this->m_reindex = reindex;
+    this->m_collectionFile = collectionFile;
+    this->m_indexFilesFolder = indexFilesFolder;
+    mutex.unlock();
+
+    start(QThread::NormalPriority);
+}
+
+void QHelpSearchIndexWriter::optimizeIndex()
+{
+    if (QCLuceneIndexReader::indexExists(m_indexFilesFolder)) {
+        if (QCLuceneIndexReader::isLocked(m_indexFilesFolder))
+            return;
+
+        QCLuceneStandardAnalyzer analyzer;
+        QCLuceneIndexWriter writer(m_indexFilesFolder, analyzer, false);
+        writer.optimize();
+        writer.close();
+    }
+}
+
+void QHelpSearchIndexWriter::run()
+{
+    QMutexLocker mutexLocker(&mutex);
+
+    if (m_cancel)
+        return;
+
+    const bool reindex = this->m_reindex;
+    const QString collectionFile(this->m_collectionFile);
+
+    mutexLocker.unlock();
+
+    QHelpEngineCore engine(collectionFile, 0);
+    if (!engine.setupData())
+        return;
+
+    const QLatin1String key("CluceneIndexedNamespaces");
+    if (reindex)
+        engine.setCustomValue(key, QLatin1String(""));
+
+    QMap<QString, QDateTime> indexMap;
+    const QLatin1String oldKey("CluceneSearchNamespaces");
+    if (!engine.customValue(oldKey, QString()).isNull()) {
+        // old style qhc file < 4.4.2, need to convert...
+        const QStringList indexedNamespaces = engine.customValue(oldKey).
+            toString().split(QLatin1String("|"), QString::SkipEmptyParts);
+        foreach (const QString& nameSpace, indexedNamespaces)
+            indexMap.insert(nameSpace, QDateTime());
+        engine.removeCustomValue(oldKey);
+    } else {
+        QDataStream dataStream(engine.customValue(key).toByteArray());
+        dataStream >> indexMap;
+    }
+
+    QString indexPath = m_indexFilesFolder;
+
+    QFileInfo fInfo(indexPath);
+    if (fInfo.exists() && !fInfo.isWritable()) {
+        qWarning("Full Text Search, could not create index (missing permissions).");
+        return;
+    }
+
+    emit indexingStarted();
+
+    QCLuceneIndexWriter *writer = 0;
+    QCLuceneStandardAnalyzer analyzer;
+    const QStringList registeredDocs = engine.registeredDocumentations();
+
+    QLocalSocket localSocket;
+    localSocket.connectToServer(QString(QLatin1String("QtAssistant%1"))
+        .arg(QLatin1String(QT_VERSION_STR)));
+
+    QLocalServer localServer;
+    bool otherInstancesRunning = true;
+    if (!localSocket.waitForConnected()) {
+        otherInstancesRunning = false;
+        localServer.listen(QString(QLatin1String("QtAssistant%1"))
+            .arg(QLatin1String(QT_VERSION_STR)));
+    }
+
+#if !defined(QT_NO_EXCEPTIONS)
+    try {
+#endif
+        // check if it's locked, and if the other instance is running
+        if (!otherInstancesRunning && QCLuceneIndexReader::isLocked(indexPath))
+            QCLuceneIndexReader::unlock(indexPath);
+
+        if (QCLuceneIndexReader::isLocked(indexPath)) {
+            // poll unless indexing finished to fake progress
+            while (QCLuceneIndexReader::isLocked(indexPath)) {
+                mutexLocker.relock();
+                if (m_cancel)
+                    break;
+                mutexLocker.unlock();
+                this->sleep(1);
+            }
+            emit indexingFinished();
+            return;
+        }
+
+        if (QCLuceneIndexReader::indexExists(indexPath) && !reindex) {
+            foreach(const QString& namespaceName, registeredDocs) {
+                mutexLocker.relock();
+                if (m_cancel) {
+                    emit indexingFinished();
+                    return;
+                }
+                mutexLocker.unlock();
+
+                if (!indexMap.contains(namespaceName)) {
+                    // make sure we remove some partly indexed stuff
+                    removeDocuments(indexPath, namespaceName);
+                } else {
+                    QString path = engine.documentationFileName(namespaceName);
+                    if (indexMap.value(namespaceName) < QFileInfo(path).lastModified()) {
+                        // make sure we remove some outdated indexed stuff
+                        indexMap.remove(namespaceName);
+                        removeDocuments(indexPath, namespaceName);
+                    }
+
+                    if (indexMap.contains(namespaceName)) {
+                        // make sure we really have content indexed for namespace
+                        // NOTE: Extra variable just for GCC 3.3.5
+                        QLatin1String key("namespace");
+                        QCLuceneTermQuery query(QCLuceneTerm(key, namespaceName));
+                        QCLuceneIndexSearcher indexSearcher(indexPath);
+                        QCLuceneHits hits = indexSearcher.search(query);
+                        if (hits.length() <= 0)
+                            indexMap.remove(namespaceName);
+                    }
+                }
+            }
+            writer = new QCLuceneIndexWriter(indexPath, analyzer, false);
+        } else {
+            indexMap.clear();
+            writer = new QCLuceneIndexWriter(indexPath, analyzer, true);
+        }
+#if !defined(QT_NO_EXCEPTIONS)
+    } catch (...) {
+        qWarning("Full Text Search, could not create index writer.");
+        return;
+    }
+#endif
+
+    writer->setMergeFactor(100);
+    writer->setMinMergeDocs(1000);
+    writer->setMaxFieldLength(QCLuceneIndexWriter::DEFAULT_MAX_FIELD_LENGTH);
+
+    QStringList namespaces;
+    foreach(const QString& namespaceName, registeredDocs) {
+        mutexLocker.relock();
+        if (m_cancel) {
+            writer->close();
+            delete writer;
+            emit indexingFinished();
+            return;
+        }
+        mutexLocker.unlock();
+
+        namespaces.append(namespaceName);
+        if (indexMap.contains(namespaceName))
+            continue;
+
+        const QList<QStringList> attributeSets =
+            engine.filterAttributeSets(namespaceName);
+
+        if (attributeSets.isEmpty()) {
+            const QList<QUrl> docFiles = indexableFiles(&engine, namespaceName,
+                QStringList());
+            if (!addDocuments(docFiles, engine, QStringList(), namespaceName,
+                writer, analyzer))
+                break;
+        } else {
+            bool bail = false;
+            foreach (const QStringList& attributes, attributeSets) {
+                const QList<QUrl> docFiles = indexableFiles(&engine,
+                    namespaceName, attributes);
+                if (!addDocuments(docFiles, engine, attributes, namespaceName,
+                    writer, analyzer)) {
+                    bail = true;
+                    break;
+                }
+            }
+            if (bail)
+                break;
+        }
+
+        mutexLocker.relock();
+        if (!m_cancel) {
+            QString path(engine.documentationFileName(namespaceName));
+            indexMap.insert(namespaceName, QFileInfo(path).lastModified());
+            writeIndexMap(engine, indexMap);
+        }
+        mutexLocker.unlock();
+    }
+
+    writer->close();
+    delete writer;
+
+    mutexLocker.relock();
+    if (!m_cancel) {
+        mutexLocker.unlock();
+
+        QStringList indexedNamespaces = indexMap.keys();
+        foreach(const QString& namespaceName, indexedNamespaces) {
+            mutexLocker.relock();
+            if (m_cancel)
+                break;
+            mutexLocker.unlock();
+
+            if (!namespaces.contains(namespaceName)) {
+                indexMap.remove(namespaceName);
+                writeIndexMap(engine, indexMap);
+                removeDocuments(indexPath, namespaceName);
+            }
+        }
+    }
+    emit indexingFinished();
+}
+
+bool QHelpSearchIndexWriter::addDocuments(const QList<QUrl> docFiles,
+    const QHelpEngineCore &engine, const QStringList &attributes,
+    const QString &namespaceName, QCLuceneIndexWriter *writer,
+    QCLuceneAnalyzer &analyzer)
+{
+    QMutexLocker locker(&mutex);
+    const QString attrList = attributes.join(QLatin1String(" "));
+
+    locker.unlock();
+    foreach(const QUrl& url, docFiles) {
+        QCLuceneDocument document;
+        DocumentHelper helper(url.toString(), engine.fileData(url));
+        if (helper.addFieldsToDocument(&document, namespaceName, attrList))
+            writer->addDocument(document, analyzer);
+
+        locker.relock();
+        if (m_cancel)
+            return false;
+        locker.unlock();
+    }
+
+    return true;
+}
+
+void QHelpSearchIndexWriter::removeDocuments(const QString &indexPath,
+    const QString &namespaceName)
+{
+    if (namespaceName.isEmpty() || QCLuceneIndexReader::isLocked(indexPath))
+        return;
+
+    QCLuceneIndexReader reader = QCLuceneIndexReader::open(indexPath);
+    reader.deleteDocuments(QCLuceneTerm(QLatin1String("namespace"),
+        namespaceName));
+
+    reader.close();
+}
+
+bool QHelpSearchIndexWriter::writeIndexMap(QHelpEngineCore& engine,
+    const QMap<QString, QDateTime>& indexMap)
+{
+    QByteArray bArray;
+
+    QDataStream data(&bArray, QIODevice::ReadWrite);
+    data << indexMap;
+
+    return engine.setCustomValue(QLatin1String("CluceneIndexedNamespaces"),
+        bArray);
+}
+
+QList<QUrl> QHelpSearchIndexWriter::indexableFiles(QHelpEngineCore *helpEngine,
+    const QString &namespaceName, const QStringList &attributes) const
+{
+    QList<QUrl> docFiles = helpEngine->files(namespaceName, attributes,
+        QLatin1String("html"));
+    docFiles += helpEngine->files(namespaceName, attributes, QLatin1String("htm"));
+    docFiles += helpEngine->files(namespaceName, attributes, QLatin1String("txt"));
+
+    return docFiles;
+}
+
+
+        }   // namespace clucene
+    }   // namespace fulltextsearch
+}   // namespace qt
+
+QT_END_NAMESPACE