Merge branch '4.5'

Conflicts:
	tools/linguist/lupdate/qscript.cpp
	tools/linguist/lupdate/qscript.g
	tools/linguist/lupdate/ui.cpp
	tools/linguist/shared/cpp.cpp
	tools/linguist/shared/qph.cpp
	tools/linguist/shared/translator.h

16 files changed, 372 insertions, 255 deletions

diff --git a/tools/linguist/linguist/mainwindow.cpp b/tools/linguist/linguist/mainwindow.cpp
index e0a8a56..6ee0dd8 100644
--- a/tools/linguist/linguist/mainwindow.cpp
+++ b/tools/linguist/linguist/mainwindow.cpp
@@ -188,7 +188,7 @@ private:
 
 static const QVariant &pxObsolete()
 {
-    static const QVariant v = 
+    static const QVariant v =
         qVariantFromValue(QPixmap(QLatin1String(":/images/s_check_obsolete.png")));
     return v;
 }
diff --git a/tools/linguist/linguist/messageeditor.cpp b/tools/linguist/linguist/messageeditor.cpp
index d1a251d..e858bbb 100644
--- a/tools/linguist/linguist/messageeditor.cpp
+++ b/tools/linguist/linguist/messageeditor.cpp
@@ -67,7 +67,7 @@ QT_BEGIN_NAMESPACE
 // Allow translators to provide localized names for QLocale::languageToString
 // At least the own language should be translated ... This is a "hack" until
 // functionality is provided within Qt (see task 196275).
-static const char * language_strings[] = 
+static const char * language_strings[] =
 {
     QT_TRANSLATE_NOOP("MessageEditor", "German"),
     QT_TRANSLATE_NOOP("MessageEditor", "Japanese"),
diff --git a/tools/linguist/linguist/messagemodel.cpp b/tools/linguist/linguist/messagemodel.cpp
index a7053cf..b049ab9 100644
--- a/tools/linguist/linguist/messagemodel.cpp
+++ b/tools/linguist/linguist/messagemodel.cpp
@@ -460,7 +460,7 @@ QString DataModel::prettifyFileName(const QString &fn)
 
 /******************************************************************************
  *
- * DataModelIterator 
+ * DataModelIterator
  *
  *****************************************************************************/
 
@@ -1109,7 +1109,7 @@ void MultiDataModel::updateCountsOnRemove(int model, bool writable)
 
 /******************************************************************************
  *
- * MultiDataModelIterator 
+ * MultiDataModelIterator
  *
  *****************************************************************************/
 
@@ -1214,17 +1214,17 @@ int MessageModel::columnCount(const QModelIndex &) const
 
 QVariant MessageModel::data(const QModelIndex &index, int role) const
 {
-    static QVariant pxOn  = 
+    static QVariant pxOn  =
         qVariantFromValue(QPixmap(QLatin1String(":/images/s_check_on.png")));
-    static QVariant pxOff = 
+    static QVariant pxOff =
         qVariantFromValue(QPixmap(QLatin1String(":/images/s_check_off.png")));
-    static QVariant pxObsolete =    
+    static QVariant pxObsolete =
         qVariantFromValue(QPixmap(QLatin1String(":/images/s_check_obsolete.png")));
     static QVariant pxDanger =
         qVariantFromValue(QPixmap(QLatin1String(":/images/s_check_danger.png")));
     static QVariant pxWarning =
         qVariantFromValue(QPixmap(QLatin1String(":/images/s_check_warning.png")));
-    static QVariant pxEmpty = 
+    static QVariant pxEmpty =
         qVariantFromValue(QPixmap(QLatin1String(":/images/s_check_empty.png")));
 
     int row = index.row();
diff --git a/tools/linguist/linguist/phraseview.cpp b/tools/linguist/linguist/phraseview.cpp
index 78c9151..72c27dc 100644
--- a/tools/linguist/linguist/phraseview.cpp
+++ b/tools/linguist/linguist/phraseview.cpp
@@ -135,7 +135,7 @@ void PhraseView::contextMenuEvent(QContextMenuEvent *event)
 void PhraseView::mouseDoubleClickEvent(QMouseEvent *event)
 {
     QModelIndex index = indexAt(event->pos());
-    if (!index.isValid()) 
+    if (!index.isValid())
         return;
 
     emit phraseSelected(m_modelIndex, m_phraseModel->phrase(index)->target());
diff --git a/tools/linguist/linguist/translationsettingsdialog.h b/tools/linguist/linguist/translationsettingsdialog.h
index 8a633d9..a7ed1b4 100644
--- a/tools/linguist/linguist/translationsettingsdialog.h
+++ b/tools/linguist/linguist/translationsettingsdialog.h
@@ -76,4 +76,4 @@ private:
 
 QT_END_NAMESPACE
 
-#endif // TRANSLATIONSETTINGSDIALOG_H 
+#endif // TRANSLATIONSETTINGSDIALOG_H
diff --git a/tools/linguist/lupdate/main.cpp b/tools/linguist/lupdate/main.cpp
index dd890ce..b241857 100644
--- a/tools/linguist/lupdate/main.cpp
+++ b/tools/linguist/lupdate/main.cpp
@@ -74,7 +74,7 @@ static void recursiveFileInfoList(const QDir &dir,
         if (fname != QLatin1String(".") && fname != QLatin1String("..")) {
             if (it->isDir())
                 recursiveFileInfoList(QDir(it->absoluteFilePath()), nameFilters, filter, recursive, fileinfolist);
-            else 
+            else
                 fileinfolist->append(*it);
         }
     }
diff --git a/tools/linguist/lupdate/merge.cpp b/tools/linguist/lupdate/merge.cpp
index 46140de..c4f4448 100644
--- a/tools/linguist/lupdate/merge.cpp
+++ b/tools/linguist/lupdate/merge.cpp
@@ -69,7 +69,7 @@ static int numberLength(const QString &s, int i)
     int pos = i;
     do {
         ++i;
-    } while (i < s.size() 
+    } while (i < s.size()
              && (s.at(i).isDigit()
                  || (isDigitFriendly(s[i])
                      && i + 1 < s.size()
@@ -445,7 +445,7 @@ Translator merge(const Translator &tor, const Translator &virginTor,
         if (mv.sourceText().isEmpty()) {
             if (tor.contains(mv.context()))
                 continue;
-        } else { 
+        } else {
             if (tor.contains(mv.context(), mv.sourceText(), mv.comment()))
                 continue;
             if (options & HeuristicSimilarText) {
diff --git a/tools/linguist/shared/po.cpp b/tools/linguist/shared/po.cpp
index e9375e9..5842771 100644
--- a/tools/linguist/shared/po.cpp
+++ b/tools/linguist/shared/po.cpp
@@ -389,7 +389,7 @@ bool loadPO(Translator &translator, QIODevice &dev, ConversionData &cd)
     for (; l != lines.size(); ++l) {
         QString line = lines.at(l);
         if (line.isEmpty())
-           continue; 
+           continue;
         if (isTranslationLine(line)) {
             bool isObsolete = line.startsWith(QLatin1String("#~ msgstr"));
             const QString prefix = QLatin1String(isObsolete ? "#~ " : "");
diff --git a/tools/linguist/shared/profileevaluator.cpp b/tools/linguist/shared/profileevaluator.cpp
index b9984cc..173b46e 100644
--- a/tools/linguist/shared/profileevaluator.cpp
+++ b/tools/linguist/shared/profileevaluator.cpp
@@ -913,7 +913,7 @@ QStringList ProFileEvaluator::Private::qmakeFeaturePaths()
     //        if (!specdir.cdUp() || specdir.isRoot())
     //            break;
     //        if (QFile::exists(specdir.path() + QDir::separator() + "features")) {
-    //            foreach (const QString &concat_it, concat) 
+    //            foreach (const QString &concat_it, concat)
     //                feature_roots << (specdir.path() + concat_it);
     //            break;
     //        }
@@ -1808,7 +1808,7 @@ bool ProFileEvaluator::Private::evaluateConditionalFunction(const QString &funct
                     parents.append(proFile->fileName());
                 if (!parents.isEmpty())
                     parents.takeLast();
-                if (parents.isEmpty()) 
+                if (parents.isEmpty())
                     q->fileMessage(format("Project ERROR: %1").arg(msg));
                 else
                     q->fileMessage(format("Project ERROR: %1. File was included from: '%2'")
diff --git a/tools/linguist/shared/qm.cpp b/tools/linguist/shared/qm.cpp
index c197e2b..d19b519 100644
--- a/tools/linguist/shared/qm.cpp
+++ b/tools/linguist/shared/qm.cpp
@@ -103,6 +103,43 @@ static uint elfHash(const QByteArray &ba)
     return h;
 }
 
+class ByteTranslatorMessage
+{
+public:
+    ByteTranslatorMessage(
+            const QByteArray &context,
+            const QByteArray &sourceText,
+            const QByteArray &comment,
+            const QStringList &translations) :
+        m_context(context),
+        m_sourcetext(sourceText),
+        m_comment(comment),
+        m_translations(translations)
+    {}
+    const QByteArray &context() const { return m_context; }
+    const QByteArray &sourceText() const { return m_sourcetext; }
+    const QByteArray &comment() const { return m_comment; }
+    const QStringList &translations() const { return m_translations; }
+    bool operator<(const ByteTranslatorMessage& m) const;
+
+private:
+    QByteArray m_context;
+    QByteArray m_sourcetext;
+    QByteArray m_comment;
+    QStringList m_translations;
+};
+
+Q_DECLARE_TYPEINFO(ByteTranslatorMessage, Q_MOVABLE_TYPE);
+
+bool ByteTranslatorMessage::operator<(const ByteTranslatorMessage& m) const
+{
+    if (m_context != m.m_context)
+        return m_context < m.m_context;
+    if (m_sourcetext != m.m_sourcetext)
+        return m_sourcetext < m.m_sourcetext;
+    return m_comment < m.m_comment;
+}
+
 class Releaser
 {
 public:
@@ -133,27 +170,12 @@ public:
         m_codec = QTextCodec::codecForName(codecName);
     }
 
-    TranslatorMessage findMessage(const QString &context,
-        const QString &sourceText, const QString &comment,
-        const QString &fileName = QString(), int lineNumber = -1) const;
-
     bool save(QIODevice *iod);
 
-    void insert(const TranslatorMessage &);
-    void remove(const TranslatorMessage &);
-
-    bool contains(const QString &context, const QString &sourceText,
-        const QString & comment) const;
-
-    bool contains(const QString &context, const QString &comment,
-        const QString &fileName, int lineNumber) const;
+    void insert(const TranslatorMessage &msg, bool forceComment);
 
     void squeeze(TranslatorSaveMode mode);
 
-    QList<TranslatorMessage> messages() const;
-
-    bool isEmpty() const;
-
     void setNumerusRules(const QByteArray &rules);
 
 private:
@@ -163,18 +185,20 @@ private:
     // on turn should be the same as passed to the actual tr(...) calls
     QByteArray originalBytes(const QString &str, bool isUtf8) const;
 
-    Prefix commonPrefix(const TranslatorMessage &m1, const TranslatorMessage &m2) const;
+    void insertInternal(const TranslatorMessage &message, bool forceComment, bool isUtf8);
+
+    static Prefix commonPrefix(const ByteTranslatorMessage &m1, const ByteTranslatorMessage &m2);
 
-    uint msgHash(const TranslatorMessage &msg) const;
+    static uint msgHash(const ByteTranslatorMessage &msg);
 
-    void writeMessage(const TranslatorMessage & msg, QDataStream & stream,
+    void writeMessage(const ByteTranslatorMessage & msg, QDataStream & stream,
         TranslatorSaveMode strip, Prefix prefix) const;
 
     // for squeezed but non-file data, this is what needs to be deleted
     QByteArray m_messageArray;
     QByteArray m_offsetArray;
     QByteArray m_contextArray;
-    QMap<TranslatorMessage, void *> m_messages;
+    QMap<ByteTranslatorMessage, void *> m_messages;
     QByteArray m_numerusRules;
 
     // Used to reproduce the original bytes
@@ -193,12 +217,12 @@ QByteArray Releaser::originalBytes(const QString &str, bool isUtf8) const
     return m_codec ? m_codec->fromUnicode(str) : str.toLatin1();
 }
 
-uint Releaser::msgHash(const TranslatorMessage &msg) const
+uint Releaser::msgHash(const ByteTranslatorMessage &msg)
 {
-    return elfHash(originalBytes(msg.sourceText() + msg.comment(), msg.isUtf8()));
+    return elfHash(msg.sourceText() + msg.comment());
 }
 
-Prefix Releaser::commonPrefix(const TranslatorMessage &m1, const TranslatorMessage &m2) const
+Prefix Releaser::commonPrefix(const ByteTranslatorMessage &m1, const ByteTranslatorMessage &m2)
 {
     if (msgHash(m1) != msgHash(m2))
         return NoPrefix;
@@ -211,7 +235,7 @@ Prefix Releaser::commonPrefix(const TranslatorMessage &m1, const TranslatorMessa
     return HashContextSourceTextComment;
 }
 
-void Releaser::writeMessage(const TranslatorMessage & msg, QDataStream & stream,
+void Releaser::writeMessage(const ByteTranslatorMessage &msg, QDataStream &stream,
     TranslatorSaveMode mode, Prefix prefix) const
 {
     for (int i = 0; i < msg.translations().count(); ++i) {
@@ -228,14 +252,14 @@ void Releaser::writeMessage(const TranslatorMessage & msg, QDataStream & stream,
     switch (prefix) {
     default:
     case HashContextSourceTextComment:
-        stream << quint8(Tag_Comment) << originalBytes(msg.comment(), msg.isUtf8());
+        stream << quint8(Tag_Comment) << msg.comment();
         // fall through
     case HashContextSourceText:
-        stream << quint8(Tag_SourceText) << originalBytes(msg.sourceText(), msg.isUtf8());
+        stream << quint8(Tag_SourceText) << msg.sourceText();
         // fall through
     case HashContext:
-        stream << quint8(Tag_Context) << originalBytes(msg.context(), msg.isUtf8());
-        ;
+        stream << quint8(Tag_Context) << msg.context();
+        break;
     }
 
     stream << quint8(Tag_End);
@@ -275,7 +299,7 @@ void Releaser::squeeze(TranslatorSaveMode mode)
     if (m_messages.isEmpty() && mode == SaveEverything)
         return;
 
-    QMap<TranslatorMessage, void *> messages = m_messages;
+    QMap<ByteTranslatorMessage, void *> messages = m_messages;
 
     // re-build contents
     m_messageArray.clear();
@@ -286,7 +310,7 @@ void Releaser::squeeze(TranslatorSaveMode mode)
     QMap<Offset, void *> offsets;
 
     QDataStream ms(&m_messageArray, QIODevice::WriteOnly);
-    QMap<TranslatorMessage, void *>::const_iterator it, next;
+    QMap<ByteTranslatorMessage, void *>::const_iterator it, next;
     int cpPrev = 0, cpNext = 0;
     for (it = messages.constBegin(); it != messages.constEnd(); ++it) {
         cpPrev = cpNext;
@@ -310,7 +334,7 @@ void Releaser::squeeze(TranslatorSaveMode mode)
     }
 
     if (mode == SaveStripped) {
-        QMap<QString, int> contextSet;
+        QMap<QByteArray, int> contextSet;
         for (it = messages.constBegin(); it != messages.constEnd(); ++it)
             ++contextSet[it.key().context()];
 
@@ -322,10 +346,10 @@ void Releaser::squeeze(TranslatorSaveMode mode)
         else
             hTableSize = (contextSet.size() < 10000) ? 15013 : 3 * contextSet.size() / 2;
 
-        QMultiMap<int, QString> hashMap;
-        QMap<QString, int>::const_iterator c;
+        QMultiMap<int, QByteArray> hashMap;
+        QMap<QByteArray, int>::const_iterator c;
         for (c = contextSet.constBegin(); c != contextSet.constEnd(); ++c)
-            hashMap.insert(elfHash(originalBytes(c.key(), false /*FIXME*/)) % hTableSize, c.key());
+            hashMap.insert(elfHash(c.key()) % hTableSize, c.key());
 
         /*
           The contexts found in this translator are stored in a hash
@@ -360,16 +384,14 @@ void Releaser::squeeze(TranslatorSaveMode mode)
         t << quint16(0); // the entry at offset 0 cannot be used
         uint upto = 2;
 
-        QMap<int, QString>::const_iterator entry = hashMap.constBegin();
+        QMap<int, QByteArray>::const_iterator entry = hashMap.constBegin();
         while (entry != hashMap.constEnd()) {
             int i = entry.key();
             hTable[i] = quint16(upto >> 1);
 
             do {
-                QString context = entry.value();
-                QByteArray ba = context.toUtf8();
-                const char *con = ba.data();
-                uint len = uint(qstrlen(con));
+                const char *con = entry.value().constData();
+                uint len = uint(entry.value().length());
                 len = qMin(len, 255u);
                 t << quint8(len);
                 t.writeRawData(con, len);
@@ -394,68 +416,28 @@ void Releaser::squeeze(TranslatorSaveMode mode)
     }
 }
 
-bool Releaser::contains(const QString &context, const QString &sourceText,
-    const QString &comment) const
-{
-    return !findMessage(context, sourceText, comment).translation().isNull();
-}
-
-bool Releaser::contains(const QString &context, const QString &comment,
-    const QString &fileName, int lineNumber) const
-{
-    return !findMessage(context, QString(), comment, fileName, lineNumber).isNull();
-}
-
-void Releaser::insert(const TranslatorMessage &message)
-{
-    m_messages.insert(message, 0);
-}
-
-void Releaser::remove(const TranslatorMessage &message)
+void Releaser::insertInternal(const TranslatorMessage &message, bool forceComment, bool isUtf8)
 {
-    m_messages.remove(message);
-}
-
-
-TranslatorMessage Releaser::findMessage(const QString &context,
-    const QString &sourceText, const QString &comment,
-    const QString &fileName, int lineNumber) const
-{
-    if (m_messages.isEmpty()) 
-        return TranslatorMessage();
-
-    QMap<TranslatorMessage, void *>::const_iterator it;
-
-    // Either we want to find an item that matches context, sourcetext
-    // (and optionally comment) Or we want to find an item that
-    // matches context, filename, linenumber (and optionally comment)
-    TranslatorMessage msg(context, sourceText, comment, QString(), fileName, lineNumber);
-    it = m_messages.constFind(msg);
-    if (it != m_messages.constEnd())
-        return it.key();
-
-    if (!comment.isEmpty()) {
-        it = m_messages.constFind(TranslatorMessage(context, sourceText, QString(), QString(), fileName, lineNumber));
-        if (it != m_messages.constEnd())
-            return it.key();
+    ByteTranslatorMessage bmsg(originalBytes(message.context(), isUtf8),
+                               originalBytes(message.sourceText(), isUtf8),
+                               originalBytes(message.comment(), isUtf8),
+                               message.translations());
+    if (!forceComment) {
+        ByteTranslatorMessage bmsg2(
+                bmsg.context(), bmsg.sourceText(), QByteArray(""), bmsg.translations());
+        if (!m_messages.contains(bmsg2)) {
+            m_messages.insert(bmsg2, 0);
+            return;
+        }
     }
-
-    it = m_messages.constFind(TranslatorMessage(context, QString(), comment, QString(), fileName, lineNumber));
-    if (it != m_messages.constEnd())
-        return it.key();
-    if (comment.isEmpty())
-        return TranslatorMessage();
-
-    it = m_messages.constFind(TranslatorMessage(context, QString(), QString(), QString(), fileName, lineNumber));
-    if (it != m_messages.constEnd())
-        return it.key();
-    return TranslatorMessage();
+    m_messages.insert(bmsg, 0);
 }
 
-bool Releaser::isEmpty() const
+void Releaser::insert(const TranslatorMessage &message, bool forceComment)
 {
-    return m_messageArray.isEmpty() && m_offsetArray.isEmpty()
-        && m_contextArray.isEmpty() && m_messages.isEmpty();
+    insertInternal(message, forceComment, message.isUtf8());
+    if (message.isUtf8() && message.isNonUtf8())
+        insertInternal(message, forceComment, false);
 }
 
 void Releaser::setNumerusRules(const QByteArray &rules)
@@ -463,11 +445,6 @@ void Releaser::setNumerusRules(const QByteArray &rules)
     m_numerusRules = rules;
 }
 
-QList<TranslatorMessage> Releaser::messages() const
-{
-    return m_messages.keys();
-}
-
 static quint8 read8(const uchar *data)
 {
     return *data;
@@ -478,6 +455,31 @@ static quint32 read32(const uchar *data)
     return (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | (data[3]);
 }
 
+static void fromBytes(const char *str, int len, QTextCodec *codec, QTextCodec *utf8Codec,
+                      QString *out, QString *utf8Out,
+                      bool *isSystem, bool *isUtf8, bool *needs8Bit)
+{
+    for (int i = 0; i < len; ++i)
+        if (str[i] & 0x80) {
+            if (utf8Codec) {
+                QTextCodec::ConverterState cvtState;
+                *utf8Out = utf8Codec->toUnicode(str, len, &cvtState);
+                *isUtf8 = !cvtState.invalidChars;
+            }
+            QTextCodec::ConverterState cvtState;
+            *out = codec->toUnicode(str, len, &cvtState);
+            *isSystem = !cvtState.invalidChars;
+            *needs8Bit = true;
+            return;
+        }
+    *out = QString::fromLatin1(str, len);
+    *isSystem = true;
+    if (utf8Codec) {
+        *utf8Out = *out;
+        *isUtf8 = true;
+    }
+    *needs8Bit = false;
+}
 
 bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
 {
@@ -543,10 +545,19 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
     size_t numItems = offsetLength / (2 * sizeof(quint32));
     //qDebug() << "NUMITEMS: " << numItems;
 
-    TranslatorMessage msg;
-
     // FIXME: that's just a guess, the original locale data is lost...
     QTextCodec *codec = QTextCodec::codecForLocale();
+    QTextCodec *utf8Codec = 0;
+    if (codec->name() != "UTF-8")
+        utf8Codec = QTextCodec::codecForName("UTF-8");
+
+    QString context, contextUtf8;
+    bool contextIsSystem, contextIsUtf8, contextNeeds8Bit;
+    QString sourcetext, sourcetextUtf8;
+    bool sourcetextIsSystem, sourcetextIsUtf8, sourcetextNeeds8Bit;
+    QString comment, commentUtf8;
+    bool commentIsSystem, commentIsUtf8, commentNeeds8Bit;
+    QStringList translations;
 
     for (const uchar *start = offsetArray; start != offsetArray + (numItems << 3); start += 8) {
         //quint32 hash = read32(start);
@@ -575,7 +586,7 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
                 }
                 str.replace(QChar(Translator::InternalVariantSeparator),
                             QChar(Translator::DefaultVariantSeparator));
-                msg.appendTranslation(str);
+                translations << str;
                 m += len;
                 break;
             }
@@ -588,7 +599,9 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
                 m += 4;
                 //qDebug() << "SOURCE LEN: " << len;
                 //qDebug() << "SOURCE: " << QByteArray((const char*)m, len);
-                msg.setSourceText(codec->toUnicode(QByteArray((const char*)m, len)));
+                fromBytes((const char*)m, len, codec, utf8Codec,
+                          &sourcetext, &sourcetextUtf8,
+                          &sourcetextIsSystem, &sourcetextIsUtf8, &sourcetextNeeds8Bit);
                 m += len;
                 break;
             }
@@ -597,7 +610,9 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
                 m += 4;
                 //qDebug() << "CONTEXT LEN: " << len;
                 //qDebug() << "CONTEXT: " << QByteArray((const char*)m, len);
-                msg.setContext(codec->toUnicode(QByteArray((const char*)m, len)));
+                fromBytes((const char*)m, len, codec, utf8Codec,
+                          &context, &contextUtf8,
+                          &contextIsSystem, &contextIsUtf8, &contextNeeds8Bit);
                 m += len;
                 break;
             }
@@ -606,7 +621,9 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
                 m += 4;
                 //qDebug() << "COMMENT LEN: " << len;
                 //qDebug() << "COMMENT: " << QByteArray((const char*)m, len);
-                msg.setComment(codec->toUnicode(QByteArray((const char*)m, len)));
+                fromBytes((const char*)m, len, codec, utf8Codec,
+                          &comment, &commentUtf8,
+                          &commentIsSystem, &commentIsUtf8, &commentNeeds8Bit);
                 m += len;
                 break;
             }
@@ -616,11 +633,33 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
             }
         }
     end:;
+        TranslatorMessage msg;
         msg.setType(TranslatorMessage::Finished);
+        msg.setTranslations(translations);
+        translations.clear();
+        if (contextNeeds8Bit || sourcetextNeeds8Bit || commentNeeds8Bit) {
+            if (utf8Codec && contextIsUtf8 && sourcetextIsUtf8 && commentIsUtf8) {
+                // The message is utf-8, but file is not.
+                msg.setUtf8(true);
+                msg.setContext(contextUtf8);
+                msg.setSourceText(sourcetextUtf8);
+                msg.setComment(commentUtf8);
+                translator.append(msg);
+                continue;
+            }
+            if (!(contextIsSystem && sourcetextIsSystem && commentIsSystem)) {
+                cd.appendError(QLatin1String(
+                        "Cannot read file with current system character codec"));
+                return false;
+            }
+            // The message is 8-bit in the file's encoding (utf-8 or not).
+        }
+        msg.setContext(context);
+        msg.setSourceText(sourcetext);
+        msg.setComment(comment);
         translator.append(msg);
-        //qDebug() << "\nHASH:" << hash << msg.sourceText() << msg.context();
-        msg.setTranslations(QStringList());
     }
+    translator.resolveDualEncoded();
     return ok;
 }
 
@@ -646,39 +685,27 @@ static bool saveQM(const Translator &translator, QIODevice &dev, ConversionData
         TranslatorMessage::Type typ = msg.type();
         if (typ != TranslatorMessage::Obsolete) {
             if (typ == TranslatorMessage::Unfinished) {
-                if (msg.translation().isEmpty())
+                if (msg.translation().isEmpty()) {
                     ++untranslated;
-                else
+                    continue;
+                } else {
+                    if (cd.ignoreUnfinished())
+                        continue;
                     ++unfinished;
+                }
             } else {
                 ++finished;
             }
-            QString context = msg.context();
-            QString sourceText = msg.sourceText();
-            QString comment = msg.comment();
-            QStringList translations = msg.translations();
-
-            if (!cd.ignoreUnfinished() || typ != TranslatorMessage::Unfinished) {
-                /*
-                Drop the comment in (context, sourceText, comment),
-                unless the context is empty,
-                unless (context, sourceText, "") already exists or
-                unless we already dropped the comment of (context,
-                sourceText, comment0).
-                */
-                if (comment.isEmpty()
-                    || context.isEmpty()
-                    || translator.contains(context, sourceText, QString())
-                    || !releaser.findMessage(context, sourceText, QString()).translation()
-                            .isNull() ) {
-                    releaser.insert(msg);
-                } else {
-                    TranslatorMessage tm(context, sourceText, QString(),
-                    QString(), QString(), -1, translations);
-                    //filename and lineNumbers will be ignored from now.
-                    releaser.insert(tm);
-                }
-            }
+            // Drop the comment in (context, sourceText, comment),
+            // unless the context is empty,
+            // unless (context, sourceText, "") already exists or
+            // unless we already dropped the comment of (context,
+            // sourceText, comment0).
+            bool forceComment =
+                    msg.comment().isEmpty()
+                    || msg.context().isEmpty()
+                    || translator.contains(msg.context(), msg.sourceText(), QString());
+            releaser.insert(msg, forceComment);
         }
     }
 
diff --git a/tools/linguist/shared/qph.cpp b/tools/linguist/shared/qph.cpp
index 6982f09..799bf7d 100644
--- a/tools/linguist/shared/qph.cpp
+++ b/tools/linguist/shared/qph.cpp
@@ -133,14 +133,47 @@ static bool loadQPH(Translator &translator, QIODevice &dev, ConversionData &cd)
     return reader.read(translator);
 }
 
+static QString protect(const QString &str)
+{
+    QString result;
+    result.reserve(str.length() * 12 / 10);
+    for (int i = 0; i != str.size(); ++i) {
+        uint c = str.at(i).unicode();
+        switch (c) {
+        case '\"':
+            result += QLatin1String("&quot;");
+            break;
+        case '&':
+            result += QLatin1String("&amp;");
+            break;
+        case '>':
+            result += QLatin1String("&gt;");
+            break;
+        case '<':
+            result += QLatin1String("&lt;");
+            break;
+        case '\'':
+            result += QLatin1String("&apos;");
+            break;
+        default:
+            if (c < 0x20 && c != '\r' && c != '\n' && c != '\t')
+                result += QString(QLatin1String("&#%1;")).arg(c);
+            else // this also covers surrogates
+                result += QChar(c);
+        }
+    }
+    return result;
+}
+
 static bool saveQPH(const Translator &translator, QIODevice &dev, ConversionData &)
 {
     QTextStream t(&dev);
-    t << "<!DOCTYPE QPH><QPH>\n";
+    t.setCodec(QTextCodec::codecForName("UTF-8"));
+    t << "<!DOCTYPE QPH>\n<QPH>\n";
     foreach (const TranslatorMessage &msg, translator.messages()) {
         t << "<phrase>\n";
-        t << "    <source>" << msg.sourceText() << "</source>\n";
-        t << "    <target>" << msg.translations().join(QLatin1String("@"))
+        t << "    <source>" << protect(msg.sourceText()) << "</source>\n";
+        t << "    <target>" << protect(msg.translations().join(QLatin1String("@")))
             << "</target>\n";
         if (!msg.context().isEmpty() || !msg.comment().isEmpty())
             t << "    <definition>" << msg.context() << msg.comment()
diff --git a/tools/linguist/shared/translator.cpp b/tools/linguist/shared/translator.cpp
index 3b5e8f3..3721204 100644
--- a/tools/linguist/shared/translator.cpp
+++ b/tools/linguist/shared/translator.cpp
@@ -105,13 +105,18 @@ void Translator::extend(const TranslatorMessage &msg)
     if (index == -1) {
         m_messages.append(msg);
     } else {
-        m_messages[index].addReferenceUniq(msg.fileName(), msg.lineNumber());
+        TranslatorMessage &emsg = m_messages[index];
+        emsg.addReferenceUniq(msg.fileName(), msg.lineNumber());
         if (!msg.extraComment().isEmpty()) {
-            QString cmt = m_messages[index].extraComment();
+            QString cmt = emsg.extraComment();
             if (!cmt.isEmpty())
                 cmt.append(QLatin1String("\n----------\n"));
             cmt.append(msg.extraComment());
-            m_messages[index].setExtraComment(cmt);
+            emsg.setExtraComment(cmt);
+        }
+        if (msg.isUtf8() != emsg.isUtf8()) {
+            emsg.setUtf8(true);
+            emsg.setNonUtf8(true);
         }
     }
 }
@@ -424,6 +429,28 @@ QList<TranslatorMessage> Translator::findDuplicates() const
     return ret;
 }
 
+void Translator::resolveDualEncoded()
+{
+    QHash<TranslatorMessage, int> dups;
+    for (int i = 0; i < m_messages.count();) {
+        const TranslatorMessage &msg = m_messages.at(i);
+        QHash<TranslatorMessage, int>::ConstIterator it = dups.constFind(msg);
+        if (it != dups.constEnd()) {
+            TranslatorMessage &omsg = m_messages[*it];
+            if (omsg.isUtf8() != msg.isUtf8() && !omsg.isNonUtf8()) {
+                omsg.setUtf8(true);
+                omsg.setNonUtf8(true);
+                m_messages.removeAt(i);
+                continue;
+            }
+            // Regular dupe; will complain later
+        } else {
+            dups[msg] = i;
+        }
+        ++i;
+    }
+}
+
 // Used by lupdate to be able to search using absolute paths during merging
 void Translator::makeFileNamesAbsolute(const QDir &originalPath)
 {
@@ -544,7 +571,7 @@ void Translator::setCodecName(const QByteArray &name)
     if (!codec) {
         if (!name.isEmpty())
             qWarning("No QTextCodec for %s available. Using Latin1\n", name.constData());
-        m_codecName.clear();
+        m_codecName = "ISO-8859-1";
     } else {
         m_codecName = codec->name();
     }
diff --git a/tools/linguist/shared/translator.h b/tools/linguist/shared/translator.h
index de98e9d..0f83c1d 100644
--- a/tools/linguist/shared/translator.h
+++ b/tools/linguist/shared/translator.h
@@ -54,8 +54,6 @@
 
 QT_BEGIN_NAMESPACE
 
-Q_DECLARE_TYPEINFO(TranslatorMessage, Q_MOVABLE_TYPE);
-
 class QIODevice;
 
 // A struct of "interesting" data passed to and from the load and save routines
@@ -93,7 +91,7 @@ public:
     QSet<QString> m_projectRoots;
     QMultiHash<QString, QString> m_allCSources;
     QStringList m_includePath;
-    QStringList m_dropTags;  // tags to be dropped 
+    QStringList m_dropTags;  // tags to be dropped
     QStringList m_errors;
     bool m_verbose;
     bool m_ignoreUnfinished;
@@ -135,6 +133,7 @@ public:
     void stripIdenticalSourceTranslations();
     void dropTranslations();
     QList<TranslatorMessage> findDuplicates() const;
+    void resolveDualEncoded();
     void makeFileNamesAbsolute(const QDir &originalPath);
 
     void setCodecName(const QByteArray &name);
diff --git a/tools/linguist/shared/translatormessage.cpp b/tools/linguist/shared/translatormessage.cpp
index ab4301f..afe66fe 100644
--- a/tools/linguist/shared/translatormessage.cpp
+++ b/tools/linguist/shared/translatormessage.cpp
@@ -54,7 +54,7 @@
 QT_BEGIN_NAMESPACE
 
 TranslatorMessage::TranslatorMessage()
-  : m_lineNumber(-1), m_type(Unfinished), m_utf8(false), m_plural(false)
+  : m_lineNumber(-1), m_type(Unfinished), m_utf8(false), m_nonUtf8(false), m_plural(false)
 {
 }
 
@@ -66,7 +66,7 @@ TranslatorMessage::TranslatorMessage(const QString &context,
   : m_context(context), m_sourcetext(sourceText), m_comment(comment),
     m_userData(userData),
     m_translations(translations), m_fileName(fileName), m_lineNumber(lineNumber),
-    m_type(type), m_utf8(false), m_plural(plural)
+    m_type(type), m_utf8(false), m_nonUtf8(false), m_plural(plural)
 {
 }
 
diff --git a/tools/linguist/shared/translatormessage.h b/tools/linguist/shared/translatormessage.h
index fa37ff5..363019e 100644
--- a/tools/linguist/shared/translatormessage.h
+++ b/tools/linguist/shared/translatormessage.h
@@ -136,6 +136,8 @@ public:
     void setType(Type t) { m_type = t; }
     bool isUtf8() const { return m_utf8; } // codecForTr override
     void setUtf8(bool on) { m_utf8 = on; }
+    bool isNonUtf8() const { return m_nonUtf8; } // codecForTr override
+    void setNonUtf8(bool on) { m_nonUtf8 = on; }
     bool isPlural() const { return m_plural; }
     void setPlural(bool isplural) { m_plural = isplural; }
 
@@ -169,9 +171,12 @@ private:
 
     Type m_type;
     bool m_utf8;
+    bool m_nonUtf8;
     bool m_plural;
 };
 
+Q_DECLARE_TYPEINFO(TranslatorMessage, Q_MOVABLE_TYPE);
+
 int qHash(const TranslatorMessage &msg);
 
 QT_END_NAMESPACE
diff --git a/tools/linguist/shared/ts.cpp b/tools/linguist/shared/ts.cpp
index 2e7d40f..8a6d365 100644
--- a/tools/linguist/shared/ts.cpp
+++ b/tools/linguist/shared/ts.cpp
@@ -212,6 +212,7 @@ QString TSReader::readTransContents()
 
 bool TSReader::read(Translator &translator)
 {
+    STRING(both);
     STRING(byte);
     STRING(comment);
     STRING(context);
@@ -265,19 +266,23 @@ bool TSReader::read(Translator &translator)
             QString currentFile;
 
             QXmlStreamAttributes atts = attributes();
-            //QString version = atts.value(strversion).toString();
+            QString version = atts.value(strversion).toString();
             translator.setLanguageCode(atts.value(strlanguage).toString());
             translator.setSourceLanguageCode(atts.value(strsourcelanguage).toString());
             while (!atEnd()) {
                 readNext();
                 if (isEndElement()) {
                     // </TS> found, finish local loop
+                    if (version == QLatin1String("1.1"))
+                        translator.resolveDualEncoded();
                     break;
                 } else if (isWhiteSpace()) {
                     // ignore these, just whitespace
                 } else if (elementStarts(strdefaultcodec)) {
                     // <defaultcodec>
-                    translator.setCodecName(readElementText().toLatin1());
+                    const QString &codec = readElementText();
+                    if (!codec.isEmpty())
+                        translator.setCodecName(codec.toLatin1());
                     // </defaultcodec>
                 } else if (isStartElement()
                         && name().toString().startsWith(strextrans)) {
@@ -309,7 +314,9 @@ bool TSReader::read(Translator &translator)
                             msg.setContext(context);
                             msg.setType(TranslatorMessage::Finished);
                             msg.setPlural(attributes().value(strnumerus) == stryes);
-                            msg.setUtf8(attributes().value(strutf8) == strtrue
+                            const QStringRef &utf8Attr = attributes().value(strutf8);
+                            msg.setNonUtf8(utf8Attr == strboth);
+                            msg.setUtf8(msg.isNonUtf8() || utf8Attr == strtrue
                                  ||  attributes().value(strencoding) == strUtf8);
                             while (!atEnd()) {
                                 readNext();
@@ -373,7 +380,7 @@ bool TSReader::read(Translator &translator)
                                 } else if (elementStarts(strtranslation)) {
                                     // <translation>
                                     QXmlStreamAttributes atts = attributes();
-                                    QStringRef type = atts.value(strtype); 
+                                    QStringRef type = atts.value(strtype);
                                     if (type == strunfinished)
                                         msg.setType(TranslatorMessage::Unfinished);
                                     else if (type == strobsolete)
@@ -440,7 +447,7 @@ static QString protect(const QString &str)
     QString result;
     result.reserve(str.length() * 12 / 10);
     for (int i = 0; i != str.size(); ++i) {
-        uint c = str.at(i).unicode(); 
+        uint c = str.at(i).unicode();
         switch (c) {
         case '\"':
             result += QLatin1String("&quot;");
@@ -592,107 +599,126 @@ bool saveTS(const Translator &translator, QIODevice &dev, ConversionData &cd, in
         foreach (const TranslatorMessage &msg, messageOrder[context]) {
             //msg.dump();
 
-            t << "    <message";
-            if (!msg.id().isEmpty())
-                t << " id=\"" << msg.id() << "\"";
-            if (format == 11 && !trIsUtf8 && msg.isUtf8())
-                t << " encoding=\"UTF-8\"";
-            if (format == 20 && !trIsUtf8 && msg.isUtf8()) 
-                t << " utf8=\"true\"";
-            if (msg.isPlural())
-                t << " numerus=\"yes\"";
-            t << ">\n";
-            if (translator.locationsType() != Translator::NoLocations) {
-                QString cfile = currentFile;
-                bool first = true;
-                foreach (const TranslatorMessage::Reference &ref, msg.allReferences()) {
-                    QString fn = cd.m_targetDir.relativeFilePath(ref.fileName())
-                                .replace(QLatin1Char('\\'),QLatin1Char('/'));
-                    int ln = ref.lineNumber();
-                    QString ld;
-                    if (translator.locationsType() == Translator::RelativeLocations) {
-                        if (ln != -1) {
-                            int dlt = ln - currentLine[fn];
-                            if (dlt >= 0)
-                                ld.append(QLatin1Char('+'));
-                            ld.append(QString::number(dlt));
-                            currentLine[fn] = ln;
+            bool isUtf8 = msg.isUtf8();
+            bool second = false;
+            forever {
+
+                t << "    <message";
+                if (!msg.id().isEmpty())
+                    t << " id=\"" << msg.id() << "\"";
+                if (!trIsUtf8) {
+                    if (format == 11) {
+                        if (isUtf8)
+                            t << " encoding=\"UTF-8\"";
+                    } else {
+                        if (msg.isUtf8()) {
+                            if (msg.isNonUtf8())
+                                t << " utf8=\"both\"";
+                            else
+                                t << " utf8=\"true\"";
                         }
+                    }
+                }
+                if (msg.isPlural())
+                    t << " numerus=\"yes\"";
+                t << ">\n";
+                if (translator.locationsType() != Translator::NoLocations) {
+                    QString cfile = currentFile;
+                    bool first = true;
+                    foreach (const TranslatorMessage::Reference &ref, msg.allReferences()) {
+                        QString fn = cd.m_targetDir.relativeFilePath(ref.fileName())
+                                    .replace(QLatin1Char('\\'),QLatin1Char('/'));
+                        int ln = ref.lineNumber();
+                        QString ld;
+                        if (translator.locationsType() == Translator::RelativeLocations) {
+                            if (ln != -1) {
+                                int dlt = ln - currentLine[fn];
+                                if (dlt >= 0)
+                                    ld.append(QLatin1Char('+'));
+                                ld.append(QString::number(dlt));
+                                currentLine[fn] = ln;
+                            }
 
-                        if (fn != cfile) {
-                            if (first)
-                                currentFile = fn;
-                            cfile = fn;
+                            if (fn != cfile) {
+                                if (first)
+                                    currentFile = fn;
+                                cfile = fn;
+                            } else {
+                                fn.clear();
+                            }
+                            first = false;
                         } else {
-                            fn.clear();
+                            if (ln != -1)
+                                ld = QString::number(ln);
                         }
-                        first = false;
-                    } else {
-                        if (ln != -1)
-                            ld = QString::number(ln);
+                        t << "        <location";
+                        if (!fn.isEmpty())
+                            t << " filename=\"" << fn << "\"";
+                        if (!ld.isEmpty())
+                            t << " line=\"" << ld << "\"";
+                        t << "/>\n";
                     }
-                    t << "        <location";
-                    if (!fn.isEmpty())
-                        t << " filename=\"" << fn << "\"";
-                    if (!ld.isEmpty())
-                        t << " line=\"" << ld << "\"";
-                    t << "/>\n";
                 }
-            }
 
-            t << "        <source>"
-              << evilBytes(msg.sourceText(), msg.isUtf8(), format, codecName)
-              << "</source>\n";
+                t << "        <source>"
+                  << evilBytes(msg.sourceText(), isUtf8, format, codecName)
+                  << "</source>\n";
 
-            if (format != 11 && !msg.oldSourceText().isEmpty())
-                t << "        <oldsource>" << protect(msg.oldSourceText()) << "</oldsource>\n";
+                if (format != 11 && !msg.oldSourceText().isEmpty())
+                    t << "        <oldsource>" << protect(msg.oldSourceText()) << "</oldsource>\n";
 
-            if (!msg.comment().isEmpty()) {
-                t << "        <comment>"
-                  << evilBytes(msg.comment(), msg.isUtf8(), format, codecName)
-                  << "</comment>\n";
-            }
+                if (!msg.comment().isEmpty()) {
+                    t << "        <comment>"
+                      << evilBytes(msg.comment(), isUtf8, format, codecName)
+                      << "</comment>\n";
+                }
 
-            if (format != 11) {
+                if (format != 11) {
 
-                if (!msg.oldComment().isEmpty())
-                    t << "        <oldcomment>" << protect(msg.oldComment()) << "</oldcomment>\n";
+                    if (!msg.oldComment().isEmpty())
+                        t << "        <oldcomment>" << protect(msg.oldComment()) << "</oldcomment>\n";
 
-                if (!msg.extraComment().isEmpty())
-                    t << "        <extracomment>" << protect(msg.extraComment())
-                      << "</extracomment>\n";
+                    if (!msg.extraComment().isEmpty())
+                        t << "        <extracomment>" << protect(msg.extraComment())
+                          << "</extracomment>\n";
 
-                if (!msg.translatorComment().isEmpty())
-                    t << "        <translatorcomment>" << protect(msg.translatorComment())
-                      << "</translatorcomment>\n";
+                    if (!msg.translatorComment().isEmpty())
+                        t << "        <translatorcomment>" << protect(msg.translatorComment())
+                          << "</translatorcomment>\n";
 
-            }
+                }
 
-            t << "        <translation";
-            if (msg.type() == TranslatorMessage::Unfinished)
-                t << " type=\"unfinished\"";
-            else if (msg.type() == TranslatorMessage::Obsolete)
-                t << " type=\"obsolete\"";
-            if (msg.isPlural()) {
-                t << ">";
-                QStringList translns = translator.normalizedTranslations(msg, cd, &result);
-                for (int j = 0; j < qMax(1, translns.count()); ++j) {
-                    t << "\n            <numerusform";
-                    writeVariants(t, "            ", translns[j]);
-                    t << "</numerusform>";
+                t << "        <translation";
+                if (msg.type() == TranslatorMessage::Unfinished)
+                    t << " type=\"unfinished\"";
+                else if (msg.type() == TranslatorMessage::Obsolete)
+                    t << " type=\"obsolete\"";
+                if (msg.isPlural()) {
+                    t << ">";
+                    QStringList translns = translator.normalizedTranslations(msg, cd, &result);
+                    for (int j = 0; j < qMax(1, translns.count()); ++j) {
+                        t << "\n            <numerusform";
+                        writeVariants(t, "            ", translns[j]);
+                        t << "</numerusform>";
+                    }
+                    t << "\n        ";
+                } else {
+                    writeVariants(t, "        ", msg.translation());
                 }
-                t << "\n        ";
-            } else {
-                writeVariants(t, "        ", msg.translation());
-            }
-            t << "</translation>\n";
+                t << "</translation>\n";
+
+                if (format != 11)
+                    writeExtras(t, "        ", msg.extras(), drops);
 
-            if (format != 11)
-                writeExtras(t, "        ", msg.extras(), drops);
+                if (!msg.userData().isEmpty())
+                    t << "        <userdata>" << msg.userData() << "</userdata>\n";
+                t << "    </message>\n";
 
-            if (!msg.userData().isEmpty())
-                t << "        <userdata>" << msg.userData() << "</userdata>\n";
-            t << "    </message>\n";
+                if (format != 11 || second || !msg.isUtf8() || !msg.isNonUtf8())
+                    break;
+                isUtf8 = false;
+                second = true;
+            }
         }
         t << "</context>\n";
     }
@@ -708,12 +734,12 @@ bool loadTS(Translator &translator, QIODevice &dev, ConversionData &cd)
     return reader.read(translator);
 }
 
-bool saveTS11(const Translator &translator, QIODevice &dev, ConversionData &cd) 
+bool saveTS11(const Translator &translator, QIODevice &dev, ConversionData &cd)
 {
     return saveTS(translator, dev, cd, 11);
 }
 
-bool saveTS20(const Translator &translator, QIODevice &dev, ConversionData &cd) 
+bool saveTS20(const Translator &translator, QIODevice &dev, ConversionData &cd)
 {
     return saveTS(translator, dev, cd, 20);
 }