Fix responseText to check the charset encoding field and also to not

assume that the data is xml. The change also follows the w3c spec
more closely (e.g. check the mime type field as well for encoding,
for both responseText and responseXML).

Task-number: QTBUG-13117

4 files changed, 110 insertions, 21 deletions

diff --git a/src/declarative/qml/qdeclarativexmlhttprequest.cpp b/src/declarative/qml/qdeclarativexmlhttprequest.cpp
index ff1a0e9..d832638 100644
--- a/src/declarative/qml/qdeclarativexmlhttprequest.cpp
+++ b/src/declarative/qml/qdeclarativexmlhttprequest.cpp
@@ -965,8 +965,9 @@ public:
     QScriptValue send(QScriptValue *me, const QByteArray &);
     QScriptValue abort(QScriptValue *me);
 
-    QString responseBody() const;
+    QString responseBody();
     const QByteArray & rawResponseBody() const;
+    bool receivedXml() const;
 private slots:
     void downloadProgress(qint64);
     void error(QNetworkReply::NetworkError);
@@ -989,6 +990,15 @@ private:
     HeadersList m_headersList;
     void fillHeadersList();
 
+    bool m_gotXml;
+    QByteArray m_mime;
+    QByteArray m_charset;
+    QTextCodec *m_textCodec;
+#ifndef QT_NO_TEXTCODEC
+    QTextCodec* findTextCodec() const;
+#endif
+    void readEncoding();
+
     QScriptValue m_me; // Set to the data object while a send() is ongoing (to access the callback)
 
     QScriptValue dispatchCallback(QScriptValue *me);
@@ -1006,7 +1016,7 @@ private:
 
 QDeclarativeXMLHttpRequest::QDeclarativeXMLHttpRequest(QNetworkAccessManager *manager)
 : m_state(Unsent), m_errorFlag(false), m_sendFlag(false),
-  m_redirectCount(0), m_network(0), m_nam(manager)
+  m_redirectCount(0), m_gotXml(false), m_textCodec(0), m_network(0), m_nam(manager)
 {
 }
 
@@ -1277,6 +1287,7 @@ void QDeclarativeXMLHttpRequest::finished()
         if (cbv.isError()) printError(cbv);
     }
     m_responseEntityBody.append(m_network->readAll());
+    readEncoding();
 
     if (xhrDump()) {
         qWarning().nospace() << "XMLHttpRequest: RESPONSE " << qPrintable(m_url.toString());
@@ -1302,15 +1313,72 @@ void QDeclarativeXMLHttpRequest::finished()
 }
 
 
-QString QDeclarativeXMLHttpRequest::responseBody() const
+void QDeclarativeXMLHttpRequest::readEncoding()
+{
+    foreach (const HeaderPair &header, m_headersList) {
+        if (header.first == "content-type") {
+            int separatorIdx = header.second.indexOf(';');
+            if (separatorIdx == -1) {
+                m_mime == header.second;
+            } else {
+                m_mime = header.second.mid(0, separatorIdx);
+                int charsetIdx = header.second.indexOf("charset=");
+                if (charsetIdx != -1) {
+                    charsetIdx += 8;
+                    separatorIdx = header.second.indexOf(';', charsetIdx);
+                    m_charset = header.second.mid(charsetIdx, separatorIdx >= 0 ? separatorIdx : header.second.length());
+                }
+            }
+            break;
+        }
+    }
+
+    if (m_mime.isEmpty() || m_mime == "text/xml" || m_mime == "application/xml" || m_mime.endsWith("+xml")) 
+        m_gotXml = true;
+}
+
+bool QDeclarativeXMLHttpRequest::receivedXml() const
+{
+    return m_gotXml;
+}
+
+
+#ifndef QT_NO_TEXTCODEC
+QTextCodec* QDeclarativeXMLHttpRequest::findTextCodec() const
+{
+    QTextCodec *codec = 0;
+
+    if (!m_charset.isEmpty()) 
+        codec = QTextCodec::codecForName(m_charset);
+
+    if (!codec && m_gotXml) {
+        QXmlStreamReader reader(m_responseEntityBody);
+        reader.readNext();
+        codec = QTextCodec::codecForName(reader.documentEncoding().toString().toUtf8());
+    }
+
+    if (!codec && m_mime == "text/html") 
+        codec = QTextCodec::codecForHtml(m_responseEntityBody, 0);
+
+    if (!codec)
+        codec = QTextCodec::codecForUtfText(m_responseEntityBody, 0);
+
+    if (!codec)
+        codec = QTextCodec::codecForName("UTF-8");
+    return codec;
+}
+#endif
+
+
+QString QDeclarativeXMLHttpRequest::responseBody()
 {
-    QXmlStreamReader reader(m_responseEntityBody);
-    reader.readNext();
 #ifndef QT_NO_TEXTCODEC
-    QTextCodec *codec = QTextCodec::codecForName(reader.documentEncoding().toString().toUtf8());
-    if (codec)
-        return codec->toUnicode(m_responseEntityBody);
+    if (!m_textCodec)
+        m_textCodec = findTextCodec();
+    if (m_textCodec)
+        return m_textCodec->toUnicode(m_responseEntityBody);
 #endif
+
     return QString::fromUtf8(m_responseEntityBody);
 }
 
@@ -1571,7 +1639,8 @@ static QScriptValue qmlxmlhttprequest_responseXML(QScriptContext *context, QScri
     if (!request) 
         THROW_REFERENCE("Not an XMLHttpRequest object");
 
-    if (request->readyState() != QDeclarativeXMLHttpRequest::Loading &&
+    if (!request->receivedXml() ||
+        request->readyState() != QDeclarativeXMLHttpRequest::Loading &&
         request->readyState() != QDeclarativeXMLHttpRequest::Done)
         return engine->nullValue();
     else  
diff --git a/tests/auto/declarative/qdeclarativexmlhttprequest/data/utf16.html b/tests/auto/declarative/qdeclarativexmlhttprequest/data/utf16.html
new file mode 100644
index 0000000..b640733
--- /dev/null
+++ b/tests/auto/declarative/qdeclarativexmlhttprequest/data/utf16.html
@@ -0,0 +1 @@
+უ Σ
diff --git a/tests/auto/declarative/qdeclarativexmlhttprequest/data/utf16.qml b/tests/auto/declarative/qdeclarativexmlhttprequest/data/utf16.qml
index a54ef4a..85bff29 100644
--- a/tests/auto/declarative/qdeclarativexmlhttprequest/data/utf16.qml
+++ b/tests/auto/declarative/qdeclarativexmlhttprequest/data/utf16.qml
@@ -3,13 +3,14 @@ import Qt 4.7
 QtObject {
     property bool dataOK: false
 
+    property string fileName
     property string responseText
     property string responseXmlRootNodeValue
 
-    Component.onCompleted: {
+    function startRequest() {
         var x = new XMLHttpRequest;
 
-        x.open("GET", "utf16.xml");
+        x.open("GET", fileName);
 
         // Test to the end
         x.onreadystatechange = function() {
diff --git a/tests/auto/declarative/qdeclarativexmlhttprequest/tst_qdeclarativexmlhttprequest.cpp b/tests/auto/declarative/qdeclarativexmlhttprequest/tst_qdeclarativexmlhttprequest.cpp
index 8141fcb..ecce349 100644
--- a/tests/auto/declarative/qdeclarativexmlhttprequest/tst_qdeclarativexmlhttprequest.cpp
+++ b/tests/auto/declarative/qdeclarativexmlhttprequest/tst_qdeclarativexmlhttprequest.cpp
@@ -104,6 +104,7 @@ private slots:
     void invalidMethodUsage();
     void redirects();
     void nonUtf8();
+    void nonUtf8_data();
 
     // Attributes
     void document();
@@ -919,26 +920,43 @@ void tst_qdeclarativexmlhttprequest::responseText_data()
 
 void tst_qdeclarativexmlhttprequest::nonUtf8()
 {
+    QFETCH(QString, fileName);
+    QFETCH(QString, responseText);
+    QFETCH(QString, xmlRootNodeValue);
+
     QDeclarativeComponent component(&engine, TEST_FILE("utf16.qml"));
     QObject *object = component.create();
     QVERIFY(object != 0);
 
+    object->setProperty("fileName", fileName);
+    QMetaObject::invokeMethod(object, "startRequest");
+
+    TRY_WAIT(object->property("dataOK").toBool() == true);
+    
+    QCOMPARE(object->property("responseText").toString(), responseText);
+
+    if (!xmlRootNodeValue.isEmpty()) {
+        QString rootNodeValue = object->property("responseXmlRootNodeValue").toString();
+        QCOMPARE(rootNodeValue, xmlRootNodeValue);
+    }
+
+    delete object;
+}
+
+void tst_qdeclarativexmlhttprequest::nonUtf8_data()
+{
+    QTest::addColumn<QString>("fileName");
+    QTest::addColumn<QString>("responseText");
+    QTest::addColumn<QString>("xmlRootNodeValue");
+
     QString uc;
     uc.resize(3);
     uc[0] = QChar(0x10e3);
     uc[1] = QChar(' ');
     uc[2] = QChar(0x03a3);
-    QString xml = "<?xml version=\"1.0\" encoding=\"UTF-16\" standalone='yes'?>\n<root>\n" + uc + "\n</root>\n";
 
-    TRY_WAIT(object->property("dataOK").toBool() == true);
-
-    QString responseText = object->property("responseText").toString();
-    QCOMPARE(responseText, xml);
-
-    QString responseXmlText = object->property("responseXmlRootNodeValue").toString();
-    QCOMPARE(responseXmlText, '\n' + uc + '\n');
-
-    delete object;
+    QTest::newRow("responseText") << "utf16.html" << uc + '\n' << "";
+    QTest::newRow("responseXML") << "utf16.xml" << "<?xml version=\"1.0\" encoding=\"UTF-16\" standalone='yes'?>\n<root>\n" + uc + "\n</root>\n" << QString('\n' + uc + '\n');
 }
 
 // Test that calling hte XMLHttpRequest methods on a non-XMLHttpRequest object