duplicate message handling improvements

- lrelease will not abort on duplicates any more
- lconvert now gets noisy, but only if messages within one file are
  duplicated (combining files with identical messages is ok)
- lupdate stays silent, but will eliminate duplicates again

- consolidate handling of dual-encoded and duplicate messages - for
  performance
- instead of the format loader, now the app is responsible for calling
  the duplicate handler. this allows for the fine-grained control
  necessary for optimal performance.

Task-number: 247738

9 files changed, 77 insertions, 68 deletions

diff --git a/tools/linguist/lconvert/main.cpp b/tools/linguist/lconvert/main.cpp
index 2842bc7..9ccc60e 100644
--- a/tools/linguist/lconvert/main.cpp
+++ b/tools/linguist/lconvert/main.cpp
@@ -210,6 +210,7 @@ int main(int argc, char *argv[])
         qWarning() << qPrintable(cd.error());
         return 2;
     }
+    Translator::reportDuplicates(tr.resolveDuplicates(), inFiles[0].name, verbose);
 
     for (int i = 1; i < inFiles.size(); ++i) {
         Translator tr2;
@@ -217,6 +218,7 @@ int main(int argc, char *argv[])
             qWarning() << qPrintable(cd.error());
             return 2;
         }
+        Translator::reportDuplicates(tr2.resolveDuplicates(), inFiles[i].name, verbose);
         for (int j = 0; j < tr2.messageCount(); ++j)
             tr.replaceSorted(tr2.message(j));
     }
diff --git a/tools/linguist/linguist/messagemodel.cpp b/tools/linguist/linguist/messagemodel.cpp
index b049ab9..7a8063b 100644
--- a/tools/linguist/linguist/messagemodel.cpp
+++ b/tools/linguist/linguist/messagemodel.cpp
@@ -209,19 +209,19 @@ bool DataModel::load(const QString &fileName, bool *langGuessed, QWidget *parent
         return false;
     }
 
-    QList<TranslatorMessage> dupes = tor.findDuplicates();
+    QSet<TranslatorMessagePtr> dupes = tor.resolveDuplicates();
     if (!dupes.isEmpty()) {
         QString err = tr("<qt>Duplicate messages found in '%1':").arg(Qt::escape(fileName));
         int numdups = 0;
-        foreach (const TranslatorMessage &msg, dupes) {
+        foreach (const TranslatorMessagePtr &msg, dupes) {
             if (++numdups >= 5) {
                 err += tr("<p>[more duplicates omitted]");
                 break;
             }
             err += tr("<p>* Context: %1<br>* Source: %2")
-                    .arg(Qt::escape(msg.context()), Qt::escape(msg.sourceText()));
-            if (!msg.comment().isEmpty())
-                err += tr("<br>* Comment: %3").arg(Qt::escape(msg.comment()));
+                    .arg(Qt::escape(msg->context()), Qt::escape(msg->sourceText()));
+            if (!msg->comment().isEmpty())
+                err += tr("<br>* Comment: %3").arg(Qt::escape(msg->comment()));
         }
         QMessageBox::warning(parent, QObject::tr("Qt Linguist"), err);
     }
diff --git a/tools/linguist/lrelease/main.cpp b/tools/linguist/lrelease/main.cpp
index f1fdb3a..3bcc998 100644
--- a/tools/linguist/lrelease/main.cpp
+++ b/tools/linguist/lrelease/main.cpp
@@ -94,19 +94,6 @@ static bool loadTsFile(Translator &tor, const QString &tsFileName, bool /* verbo
     } else {
         if (!cd.errors().isEmpty())
             printOut(cd.error());
-        const QList<TranslatorMessage> dupes = tor.findDuplicates();
-        if (!dupes.isEmpty()) {
-            qWarning("lrelease error: duplicate messages found in '%s':",
-                     qPrintable(tsFileName));
-            foreach (const TranslatorMessage &msg, dupes) {
-                qWarning("\n* Context: %s\n* Source: %s",
-                        qPrintable(msg.context()),
-                        qPrintable(msg.sourceText()));
-                if (!msg.comment().isEmpty())
-                    qWarning("\n* Comment: %s", qPrintable(msg.comment()));
-            }
-            ok = false;
-        }
     }
     return ok;
 }
@@ -115,6 +102,8 @@ static bool releaseTranslator(Translator &tor, const QString &qmFileName,
     bool verbose, bool ignoreUnfinished,
     bool removeIdentical, TranslatorSaveMode mode)
 {
+    Translator::reportDuplicates(tor.resolveDuplicates(), qmFileName, verbose);
+
     if (verbose)
         printOut(QCoreApplication::tr( "Updating '%1'...\n").arg(qmFileName));
     if (removeIdentical) {
diff --git a/tools/linguist/lupdate/main.cpp b/tools/linguist/lupdate/main.cpp
index 5869838..b537b6e 100644
--- a/tools/linguist/lupdate/main.cpp
+++ b/tools/linguist/lupdate/main.cpp
@@ -146,6 +146,7 @@ static void updateTsFiles(const Translator &fetchedTor, const QStringList &tsFil
                 *fail = true;
                 continue;
             }
+            tor.resolveDuplicates();
             cd.clearErrors();
             if (!codecForTr.isEmpty() && codecForTr != tor.codecName())
                 qWarning("lupdate warning: Codec for tr() '%s' disagrees with "
diff --git a/tools/linguist/shared/qm.cpp b/tools/linguist/shared/qm.cpp
index d19b519..5563ac5 100644
--- a/tools/linguist/shared/qm.cpp
+++ b/tools/linguist/shared/qm.cpp
@@ -659,7 +659,6 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
         msg.setComment(comment);
         translator.append(msg);
     }
-    translator.resolveDualEncoded();
     return ok;
 }
 
diff --git a/tools/linguist/shared/translator.cpp b/tools/linguist/shared/translator.cpp
index 9ac1dd8..312bb71 100644
--- a/tools/linguist/shared/translator.cpp
+++ b/tools/linguist/shared/translator.cpp
@@ -416,59 +416,52 @@ void Translator::dropTranslations()
     }
 }
 
-struct TranslatorMessagePtr {
-    TranslatorMessagePtr(const TranslatorMessage &tm)
-    {
-        ptr = &tm;
-    }
-
-    const TranslatorMessage *ptr;
-};
-
-Q_DECLARE_TYPEINFO(TranslatorMessagePtr, Q_MOVABLE_TYPE);
-
-static inline int qHash(TranslatorMessagePtr tmp)
+QSet<TranslatorMessagePtr> Translator::resolveDuplicates()
 {
-    return qHash(*tmp.ptr);
-}
-
-static inline bool operator==(TranslatorMessagePtr tmp1, TranslatorMessagePtr tmp2)
-{
-    return *tmp1.ptr == *tmp2.ptr;
-}
-
-QList<TranslatorMessage> Translator::findDuplicates() const
-{
-    QHash<TranslatorMessagePtr, int> dups;
-    foreach (const TranslatorMessage &msg, m_messages)
-        dups[msg]++;
-    QList<TranslatorMessage> ret;
-    QHash<TranslatorMessagePtr, int>::ConstIterator it = dups.constBegin(), end = dups.constEnd();
-    for (; it != end; ++it)
-        if (it.value() > 1)
-            ret.append(*it.key().ptr);
-    return ret;
-}
-
-void Translator::resolveDualEncoded()
-{
-    QHash<TranslatorMessagePtr, int> dups;
+    QSet<TranslatorMessagePtr> dups;
+    QHash<TranslatorMessagePtr, int> refs;
     for (int i = 0; i < m_messages.count();) {
         const TranslatorMessage &msg = m_messages.at(i);
-        QHash<TranslatorMessagePtr, int>::ConstIterator it = dups.constFind(msg);
-        if (it != dups.constEnd()) {
+        QHash<TranslatorMessagePtr, int>::ConstIterator it = refs.constFind(msg);
+        if (it != refs.constEnd()) {
             TranslatorMessage &omsg = m_messages[*it];
             if (omsg.isUtf8() != msg.isUtf8() && !omsg.isNonUtf8()) {
+                // Dual-encoded message
                 omsg.setUtf8(true);
                 omsg.setNonUtf8(true);
-                m_messages.removeAt(i);
-                continue;
+            } else {
+                // Duplicate
+                dups.insert(omsg);
             }
-            // Regular dupe; will complain later
+            if (!omsg.isTranslated() && msg.isTranslated())
+                omsg.setTranslations(msg.translations());
+            m_messages.removeAt(i);
+        } else {
+            refs[msg] = i;
+            ++i;
+        }
+    }
+    return dups;
+}
+
+void Translator::reportDuplicates(const QSet<TranslatorMessagePtr> &dupes,
+                                  const QString &fileName, bool verbose)
+{
+    if (!dupes.isEmpty()) {
+        if (!verbose) {
+            qWarning("Warning: dropping duplicate messages in '%s'\n(try -verbose for more info).",
+                     qPrintable(fileName));
         } else {
-            dups[msg] = i;
+            qWarning("Warning: dropping duplicate messages in '%s':", qPrintable(fileName));
+            foreach (const TranslatorMessagePtr &msg, dupes) {
+                qWarning("\n* Context: %s\n* Source: %s",
+                        qPrintable(msg->context()),
+                        qPrintable(msg->sourceText()));
+                if (!msg->comment().isEmpty())
+                    qWarning("* Comment: %s", qPrintable(msg->comment()));
+            }
+            qWarning();
         }
-        ++i;
     }
 }
 
diff --git a/tools/linguist/shared/translator.h b/tools/linguist/shared/translator.h
index 8908305..6b88b23 100644
--- a/tools/linguist/shared/translator.h
+++ b/tools/linguist/shared/translator.h
@@ -127,9 +127,10 @@ public:
     void stripNonPluralForms();
     void stripIdenticalSourceTranslations();
     void dropTranslations();
-    QList<TranslatorMessage> findDuplicates() const;
-    void resolveDualEncoded();
     void makeFileNamesAbsolute(const QDir &originalPath);
+    QSet<TranslatorMessagePtr> resolveDuplicates();
+    static void reportDuplicates(const QSet<TranslatorMessagePtr> &dupes,
+                                 const QString &fileName, bool verbose);
 
     void setCodecName(const QByteArray &name);
     QByteArray codecName() const { return m_codecName; }
diff --git a/tools/linguist/shared/translatormessage.h b/tools/linguist/shared/translatormessage.h
index 363019e..7d31925 100644
--- a/tools/linguist/shared/translatormessage.h
+++ b/tools/linguist/shared/translatormessage.h
@@ -179,6 +179,32 @@ Q_DECLARE_TYPEINFO(TranslatorMessage, Q_MOVABLE_TYPE);
 
 int qHash(const TranslatorMessage &msg);
 
+struct TranslatorMessagePtr {
+    TranslatorMessagePtr(const TranslatorMessage &tm)
+    {
+        ptr = &tm;
+    }
+
+    inline const TranslatorMessage *operator->() const
+    {
+        return ptr;
+    }
+
+    const TranslatorMessage *ptr;
+};
+
+Q_DECLARE_TYPEINFO(TranslatorMessagePtr, Q_MOVABLE_TYPE);
+
+static inline int qHash(TranslatorMessagePtr tmp)
+{
+    return qHash(*tmp.ptr);
+}
+
+static inline bool operator==(TranslatorMessagePtr tmp1, TranslatorMessagePtr tmp2)
+{
+    return *tmp1.ptr == *tmp2.ptr;
+}
+
 QT_END_NAMESPACE
 
 #endif // QT_NO_TRANSLATION
diff --git a/tools/linguist/shared/ts.cpp b/tools/linguist/shared/ts.cpp
index 8a6d365..22f2a1b 100644
--- a/tools/linguist/shared/ts.cpp
+++ b/tools/linguist/shared/ts.cpp
@@ -242,7 +242,7 @@ bool TSReader::read(Translator &translator)
     STRING(userdata);
     STRING(utf8);
     STRING(value);
-    STRING(version);
+    //STRING(version);
     STRING(yes);
 
     static const QString strextrans(QLatin1String("extra-"));
@@ -266,15 +266,13 @@ bool TSReader::read(Translator &translator)
             QString currentFile;
 
             QXmlStreamAttributes atts = attributes();
-            QString version = atts.value(strversion).toString();
+            //QString version = atts.value(strversion).toString();
             translator.setLanguageCode(atts.value(strlanguage).toString());
             translator.setSourceLanguageCode(atts.value(strsourcelanguage).toString());
             while (!atEnd()) {
                 readNext();
                 if (isEndElement()) {
                     // </TS> found, finish local loop
-                    if (version == QLatin1String("1.1"))
-                        translator.resolveDualEncoded();
                     break;
                 } else if (isWhiteSpace()) {
                     // ignore these, just whitespace