properly deal with messages which appear in multiple encodings

in ts 1.1 and qm files, messages appear in their native encoding. that
means that a message can appear multiple times - once in utf8 and once
in the codecForTr. however, in ts 2.0 files, everything is utf8 and
messages can have a utf8 flag for the later transformation into qm.
unfortunately, there was no flag to mark that the message is needed in
*both* encodings, and the respective case was completely ignored when
reading ts 1.1 and qm files (causing error messages).

Task-number: 249022
AutoTest: 322690

1 files changed, 109 insertions, 85 deletions

diff --git a/tools/linguist/shared/ts.cpp b/tools/linguist/shared/ts.cpp
index 3af9e59..8a6d365 100644
--- a/tools/linguist/shared/ts.cpp
+++ b/tools/linguist/shared/ts.cpp
@@ -212,6 +212,7 @@ QString TSReader::readTransContents()
 
 bool TSReader::read(Translator &translator)
 {
+    STRING(both);
     STRING(byte);
     STRING(comment);
     STRING(context);
@@ -265,13 +266,15 @@ bool TSReader::read(Translator &translator)
             QString currentFile;
 
             QXmlStreamAttributes atts = attributes();
-            //QString version = atts.value(strversion).toString();
+            QString version = atts.value(strversion).toString();
             translator.setLanguageCode(atts.value(strlanguage).toString());
             translator.setSourceLanguageCode(atts.value(strsourcelanguage).toString());
             while (!atEnd()) {
                 readNext();
                 if (isEndElement()) {
                     // </TS> found, finish local loop
+                    if (version == QLatin1String("1.1"))
+                        translator.resolveDualEncoded();
                     break;
                 } else if (isWhiteSpace()) {
                     // ignore these, just whitespace
@@ -311,7 +314,9 @@ bool TSReader::read(Translator &translator)
                             msg.setContext(context);
                             msg.setType(TranslatorMessage::Finished);
                             msg.setPlural(attributes().value(strnumerus) == stryes);
-                            msg.setUtf8(attributes().value(strutf8) == strtrue
+                            const QStringRef &utf8Attr = attributes().value(strutf8);
+                            msg.setNonUtf8(utf8Attr == strboth);
+                            msg.setUtf8(msg.isNonUtf8() || utf8Attr == strtrue
                                  ||  attributes().value(strencoding) == strUtf8);
                             while (!atEnd()) {
                                 readNext();
@@ -594,107 +599,126 @@ bool saveTS(const Translator &translator, QIODevice &dev, ConversionData &cd, in
         foreach (const TranslatorMessage &msg, messageOrder[context]) {
             //msg.dump();
 
-            t << "    <message";
-            if (!msg.id().isEmpty())
-                t << " id=\"" << msg.id() << "\"";
-            if (format == 11 && !trIsUtf8 && msg.isUtf8())
-                t << " encoding=\"UTF-8\"";
-            if (format == 20 && !trIsUtf8 && msg.isUtf8()) 
-                t << " utf8=\"true\"";
-            if (msg.isPlural())
-                t << " numerus=\"yes\"";
-            t << ">\n";
-            if (translator.locationsType() != Translator::NoLocations) {
-                QString cfile = currentFile;
-                bool first = true;
-                foreach (const TranslatorMessage::Reference &ref, msg.allReferences()) {
-                    QString fn = cd.m_targetDir.relativeFilePath(ref.fileName())
-                                .replace(QLatin1Char('\\'),QLatin1Char('/'));
-                    int ln = ref.lineNumber();
-                    QString ld;
-                    if (translator.locationsType() == Translator::RelativeLocations) {
-                        if (ln != -1) {
-                            int dlt = ln - currentLine[fn];
-                            if (dlt >= 0)
-                                ld.append(QLatin1Char('+'));
-                            ld.append(QString::number(dlt));
-                            currentLine[fn] = ln;
+            bool isUtf8 = msg.isUtf8();
+            bool second = false;
+            forever {
+
+                t << "    <message";
+                if (!msg.id().isEmpty())
+                    t << " id=\"" << msg.id() << "\"";
+                if (!trIsUtf8) {
+                    if (format == 11) {
+                        if (isUtf8)
+                            t << " encoding=\"UTF-8\"";
+                    } else {
+                        if (msg.isUtf8()) {
+                            if (msg.isNonUtf8())
+                                t << " utf8=\"both\"";
+                            else
+                                t << " utf8=\"true\"";
                         }
+                    }
+                }
+                if (msg.isPlural())
+                    t << " numerus=\"yes\"";
+                t << ">\n";
+                if (translator.locationsType() != Translator::NoLocations) {
+                    QString cfile = currentFile;
+                    bool first = true;
+                    foreach (const TranslatorMessage::Reference &ref, msg.allReferences()) {
+                        QString fn = cd.m_targetDir.relativeFilePath(ref.fileName())
+                                    .replace(QLatin1Char('\\'),QLatin1Char('/'));
+                        int ln = ref.lineNumber();
+                        QString ld;
+                        if (translator.locationsType() == Translator::RelativeLocations) {
+                            if (ln != -1) {
+                                int dlt = ln - currentLine[fn];
+                                if (dlt >= 0)
+                                    ld.append(QLatin1Char('+'));
+                                ld.append(QString::number(dlt));
+                                currentLine[fn] = ln;
+                            }
 
-                        if (fn != cfile) {
-                            if (first)
-                                currentFile = fn;
-                            cfile = fn;
+                            if (fn != cfile) {
+                                if (first)
+                                    currentFile = fn;
+                                cfile = fn;
+                            } else {
+                                fn.clear();
+                            }
+                            first = false;
                         } else {
-                            fn.clear();
+                            if (ln != -1)
+                                ld = QString::number(ln);
                         }
-                        first = false;
-                    } else {
-                        if (ln != -1)
-                            ld = QString::number(ln);
+                        t << "        <location";
+                        if (!fn.isEmpty())
+                            t << " filename=\"" << fn << "\"";
+                        if (!ld.isEmpty())
+                            t << " line=\"" << ld << "\"";
+                        t << "/>\n";
                     }
-                    t << "        <location";
-                    if (!fn.isEmpty())
-                        t << " filename=\"" << fn << "\"";
-                    if (!ld.isEmpty())
-                        t << " line=\"" << ld << "\"";
-                    t << "/>\n";
                 }
-            }
 
-            t << "        <source>"
-              << evilBytes(msg.sourceText(), msg.isUtf8(), format, codecName)
-              << "</source>\n";
+                t << "        <source>"
+                  << evilBytes(msg.sourceText(), isUtf8, format, codecName)
+                  << "</source>\n";
 
-            if (format != 11 && !msg.oldSourceText().isEmpty())
-                t << "        <oldsource>" << protect(msg.oldSourceText()) << "</oldsource>\n";
+                if (format != 11 && !msg.oldSourceText().isEmpty())
+                    t << "        <oldsource>" << protect(msg.oldSourceText()) << "</oldsource>\n";
 
-            if (!msg.comment().isEmpty()) {
-                t << "        <comment>"
-                  << evilBytes(msg.comment(), msg.isUtf8(), format, codecName)
-                  << "</comment>\n";
-            }
+                if (!msg.comment().isEmpty()) {
+                    t << "        <comment>"
+                      << evilBytes(msg.comment(), isUtf8, format, codecName)
+                      << "</comment>\n";
+                }
 
-            if (format != 11) {
+                if (format != 11) {
 
-                if (!msg.oldComment().isEmpty())
-                    t << "        <oldcomment>" << protect(msg.oldComment()) << "</oldcomment>\n";
+                    if (!msg.oldComment().isEmpty())
+                        t << "        <oldcomment>" << protect(msg.oldComment()) << "</oldcomment>\n";
 
-                if (!msg.extraComment().isEmpty())
-                    t << "        <extracomment>" << protect(msg.extraComment())
-                      << "</extracomment>\n";
+                    if (!msg.extraComment().isEmpty())
+                        t << "        <extracomment>" << protect(msg.extraComment())
+                          << "</extracomment>\n";
 
-                if (!msg.translatorComment().isEmpty())
-                    t << "        <translatorcomment>" << protect(msg.translatorComment())
-                      << "</translatorcomment>\n";
+                    if (!msg.translatorComment().isEmpty())
+                        t << "        <translatorcomment>" << protect(msg.translatorComment())
+                          << "</translatorcomment>\n";
 
-            }
+                }
 
-            t << "        <translation";
-            if (msg.type() == TranslatorMessage::Unfinished)
-                t << " type=\"unfinished\"";
-            else if (msg.type() == TranslatorMessage::Obsolete)
-                t << " type=\"obsolete\"";
-            if (msg.isPlural()) {
-                t << ">";
-                QStringList translns = translator.normalizedTranslations(msg, cd, &result);
-                for (int j = 0; j < qMax(1, translns.count()); ++j) {
-                    t << "\n            <numerusform";
-                    writeVariants(t, "            ", translns[j]);
-                    t << "</numerusform>";
+                t << "        <translation";
+                if (msg.type() == TranslatorMessage::Unfinished)
+                    t << " type=\"unfinished\"";
+                else if (msg.type() == TranslatorMessage::Obsolete)
+                    t << " type=\"obsolete\"";
+                if (msg.isPlural()) {
+                    t << ">";
+                    QStringList translns = translator.normalizedTranslations(msg, cd, &result);
+                    for (int j = 0; j < qMax(1, translns.count()); ++j) {
+                        t << "\n            <numerusform";
+                        writeVariants(t, "            ", translns[j]);
+                        t << "</numerusform>";
+                    }
+                    t << "\n        ";
+                } else {
+                    writeVariants(t, "        ", msg.translation());
                 }
-                t << "\n        ";
-            } else {
-                writeVariants(t, "        ", msg.translation());
-            }
-            t << "</translation>\n";
+                t << "</translation>\n";
+
+                if (format != 11)
+                    writeExtras(t, "        ", msg.extras(), drops);
 
-            if (format != 11)
-                writeExtras(t, "        ", msg.extras(), drops);
+                if (!msg.userData().isEmpty())
+                    t << "        <userdata>" << msg.userData() << "</userdata>\n";
+                t << "    </message>\n";
 
-            if (!msg.userData().isEmpty())
-                t << "        <userdata>" << msg.userData() << "</userdata>\n";
-            t << "    </message>\n";
+                if (format != 11 || second || !msg.isUtf8() || !msg.isNonUtf8())
+                    break;
+                isUtf8 = false;
+                second = true;
+            }
         }
         t << "</context>\n";
     }