Assistant: Look for document encoding in XML headers as well.

Task-number:  QTBUG-1770
Reviewed-by: kh1

6 files changed, 124 insertions, 57 deletions

diff --git a/tools/assistant/lib/lib.pro b/tools/assistant/lib/lib.pro
index 322320a..26d3456 100644
--- a/tools/assistant/lib/lib.pro
+++ b/tools/assistant/lib/lib.pro
@@ -40,7 +40,8 @@ SOURCES += qhelpenginecore.cpp \
     qhelpsearchindexwriter_default.cpp \
     qhelpsearchindexreader_default.cpp \
     qhelpsearchindexreader.cpp \
-    qclucenefieldnames.cpp
+    qclucenefieldnames.cpp \
+    qhelp_global.cpp
 
 # access to clucene
 SOURCES += qhelpsearchindexwriter_clucene.cpp \
diff --git a/tools/assistant/lib/qhelp_global.cpp b/tools/assistant/lib/qhelp_global.cpp
new file mode 100644
index 0000000..980de27
--- /dev/null
+++ b/tools/assistant/lib/qhelp_global.cpp
@@ -0,0 +1,112 @@
+/****************************************************************************
+**
+** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the Qt Assistant.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights.  These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <QtCore/QRegExp>
+#include <QtCore/QMutexLocker>
+#include <QtGui/QTextDocument>
+
+#include "qhelp_global.h"
+
+QString QHelpGlobal::uniquifyConnectionName(const QString &name, void *pointer)
+{
+    static int counter = 0;
+    static QMutex mutex;
+
+    QMutexLocker locker(&mutex);
+    if (++counter > 1000)
+        counter = 0;
+
+    return QString::fromLatin1("%1-%2-%3").
+        arg(name).arg(long(pointer)).arg(counter);
+}
+
+QString QHelpGlobal::documentTitle(const QString &content)
+{
+    QString title = QObject::tr("Untitled");
+    if (!content.isEmpty()) {
+        int start = content.indexOf(QLatin1String("<title>"), 0, Qt::CaseInsensitive) + 7;
+        int end = content.indexOf(QLatin1String("</title>"), 0, Qt::CaseInsensitive);
+        if ((end - start) > 0) {
+            title = content.mid(start, end - start);
+            if (Qt::mightBeRichText(title) || title.contains(QLatin1Char('&'))) {
+                QTextDocument doc;
+                doc.setHtml(title);
+                title = doc.toPlainText();
+            }
+        }
+    }
+    return title;
+}
+
+QString QHelpGlobal::codecFromData(const QByteArray &data)
+{
+    QString codec = codecFromXmlData(data);
+    if (codec.isEmpty())
+        codec = codecFromHtmlData(data);
+    return codec.isEmpty() ? QLatin1String("utf-8") : codec;
+}
+
+QString QHelpGlobal::codecFromHtmlData(const QByteArray &data)
+{
+    QString content = QString::fromUtf8(data.constData(), data.size());
+    int start = content.indexOf(QLatin1String("<meta"), 0, Qt::CaseInsensitive);
+    if (start > 0) {
+        int end;
+        QRegExp r(QLatin1String("charset=([^\"\\s]+)"));
+        while (start != -1) {
+            end = content.indexOf(QLatin1Char('>'), start) + 1;
+            const QString &meta = content.mid(start, end - start).toLower();
+            if (r.indexIn(meta) != -1)
+                return r.cap(1);
+            start = content.indexOf(QLatin1String("<meta"), end,
+                                    Qt::CaseInsensitive);
+        }
+    }
+    return QString();
+}
+
+QString QHelpGlobal::codecFromXmlData(const QByteArray &data)
+{
+    QString content = QString::fromUtf8(data.constData(), data.size());
+    const QRegExp encodingExp(QLatin1String("^\\s*<\\?xml version="
+        "\"\\d\\.\\d\" encoding=\"([^\"]+)\"\\?>.*"));
+    return encodingExp.exactMatch(content) ? encodingExp.cap(1) : QString();
+}
diff --git a/tools/assistant/lib/qhelp_global.h b/tools/assistant/lib/qhelp_global.h
index 723d867..4e31d67 100644
--- a/tools/assistant/lib/qhelp_global.h
+++ b/tools/assistant/lib/qhelp_global.h
@@ -45,9 +45,6 @@
 #include <QtCore/qglobal.h>
 #include <QtCore/QString>
 #include <QtCore/QObject>
-#include <QtCore/QRegExp>
-#include <QtCore/QMutexLocker>
-#include <QtGui/QTextDocument>
 
 QT_BEGIN_HEADER
 
@@ -65,56 +62,13 @@ QT_MODULE(Help)
 
 class QHelpGlobal {
 public:
-    static QString uniquifyConnectionName(const QString &name, void *pointer)
-    {
-        static int counter = 0;
-        static QMutex mutex;
+    static QString uniquifyConnectionName(const QString &name, void *pointer);
+    static QString documentTitle(const QString &content);
+    static QString codecFromData(const QByteArray &data);
 
-        QMutexLocker locker(&mutex);
-        if (++counter > 1000)
-            counter = 0;
-        
-        return QString::fromLatin1("%1-%2-%3")
-            .arg(name).arg(long(pointer)).arg(counter);
-    };
-
-    static QString documentTitle(const QString &content)
-    {
-        QString title = QObject::tr("Untitled");
-        if (!content.isEmpty()) {
-            int start = content.indexOf(QLatin1String("<title>"), 0, Qt::CaseInsensitive) + 7;
-            int end = content.indexOf(QLatin1String("</title>"), 0, Qt::CaseInsensitive);
-            if ((end - start) > 0) {
-                title = content.mid(start, end - start);
-                if (Qt::mightBeRichText(title) || title.contains(QLatin1Char('&'))) {
-                    QTextDocument doc;
-                    doc.setHtml(title);
-                    title = doc.toPlainText();
-                }
-            }
-        }
-        return title;
-    };
-
-    static QString charsetFromData(const QByteArray &data)
-    {
-        QString content = QString::fromUtf8(data.constData(), data.size());
-        int start =
-            content.indexOf(QLatin1String("<meta"), 0, Qt::CaseInsensitive);
-        if (start > 0) {
-            int end;
-            QRegExp r(QLatin1String("charset=([^\"\\s]+)"));
-            while (start != -1) {
-                end = content.indexOf(QLatin1Char('>'), start) + 1;
-                const QString &meta = content.mid(start, end - start).toLower();
-                if (r.indexIn(meta) != -1)
-                    return r.cap(1);
-                start = content.indexOf(QLatin1String("<meta"), end,
-                    Qt::CaseInsensitive);
-            }
-        }
-        return QLatin1String("utf-8");
-    }
+private:
+    static QString codecFromHtmlData(const QByteArray &data);
+    static QString codecFromXmlData(const QByteArray &data);
 };
 
 QT_END_NAMESPACE
diff --git a/tools/assistant/lib/qhelpgenerator.cpp b/tools/assistant/lib/qhelpgenerator.cpp
index 0294b30..48d73aa 100644
--- a/tools/assistant/lib/qhelpgenerator.cpp
+++ b/tools/assistant/lib/qhelpgenerator.cpp
@@ -528,7 +528,7 @@ bool QHelpGenerator::insertFiles(const QStringList &files, const QString &rootPa
         QByteArray data = fi.readAll();
         if (fileName.endsWith(QLatin1String(".html"))
             || fileName.endsWith(QLatin1String(".htm"))) {
-                charSet = QHelpGlobal::charsetFromData(data);
+                charSet = QHelpGlobal::codecFromData(data);
                 QTextStream stream(&data);
                 stream.setCodec(QTextCodec::codecForName(charSet.toLatin1().constData()));
                 title = QHelpGlobal::documentTitle(stream.readAll());
diff --git a/tools/assistant/lib/qhelpsearchindexwriter_clucene.cpp b/tools/assistant/lib/qhelpsearchindexwriter_clucene.cpp
index 80ee933..f23efa5 100644
--- a/tools/assistant/lib/qhelpsearchindexwriter_clucene.cpp
+++ b/tools/assistant/lib/qhelpsearchindexwriter_clucene.cpp
@@ -431,8 +431,8 @@ private:
     QString readData(const QByteArray &data)
     {
         QTextStream textStream(data);
-        QByteArray charSet = QHelpGlobal::charsetFromData(data).toLatin1();
-        textStream.setCodec(QTextCodec::codecForName(charSet.constData()));
+        const QByteArray &codec = QHelpGlobal::codecFromData(data).toLatin1();
+        textStream.setCodec(QTextCodec::codecForName(codec.constData()));
 
         QString stream = textStream.readAll();
         if (stream.isNull() || stream.isEmpty())
diff --git a/tools/assistant/lib/qhelpsearchindexwriter_default.cpp b/tools/assistant/lib/qhelpsearchindexwriter_default.cpp
index 36b115e..06deb85 100644
--- a/tools/assistant/lib/qhelpsearchindexwriter_default.cpp
+++ b/tools/assistant/lib/qhelpsearchindexwriter_default.cpp
@@ -274,7 +274,7 @@ void QHelpSearchIndexWriter::run()
                     continue;
 
                 QTextStream s(data);
-                QString en = QHelpGlobal::charsetFromData(data);
+                QString en = QHelpGlobal::codecFromData(data);
                 s.setCodec(QTextCodec::codecForName(en.toLatin1().constData()));
 
                 QString text = s.readAll();