duplicate message handling improvements

- lrelease will not abort on duplicates any more
- lconvert now gets noisy, but only if messages within one file are
  duplicated (combining files with identical messages is ok)
- lupdate stays silent, but will eliminate duplicates again

- consolidate handling of dual-encoded and duplicate messages - for
  performance
- instead of the format loader, now the app is responsible for calling
  the duplicate handler. this allows for the fine-grained control
  necessary for optimal performance.

Task-number: 247738

5 files changed, 67 insertions, 50 deletions

diff --git a/tools/linguist/shared/qm.cpp b/tools/linguist/shared/qm.cpp
index d19b519..5563ac5 100644
--- a/tools/linguist/shared/qm.cpp
+++ b/tools/linguist/shared/qm.cpp
@@ -659,7 +659,6 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
         msg.setComment(comment);
         translator.append(msg);
     }
-    translator.resolveDualEncoded();
     return ok;
 }
 
diff --git a/tools/linguist/shared/translator.cpp b/tools/linguist/shared/translator.cpp
index 9ac1dd8..312bb71 100644
--- a/tools/linguist/shared/translator.cpp
+++ b/tools/linguist/shared/translator.cpp
@@ -416,59 +416,52 @@ void Translator::dropTranslations()
     }
 }
 
-struct TranslatorMessagePtr {
-    TranslatorMessagePtr(const TranslatorMessage &tm)
-    {
-        ptr = &tm;
-    }
-
-    const TranslatorMessage *ptr;
-};
-
-Q_DECLARE_TYPEINFO(TranslatorMessagePtr, Q_MOVABLE_TYPE);
-
-static inline int qHash(TranslatorMessagePtr tmp)
+QSet<TranslatorMessagePtr> Translator::resolveDuplicates()
 {
-    return qHash(*tmp.ptr);
-}
-
-static inline bool operator==(TranslatorMessagePtr tmp1, TranslatorMessagePtr tmp2)
-{
-    return *tmp1.ptr == *tmp2.ptr;
-}
-
-QList<TranslatorMessage> Translator::findDuplicates() const
-{
-    QHash<TranslatorMessagePtr, int> dups;
-    foreach (const TranslatorMessage &msg, m_messages)
-        dups[msg]++;
-    QList<TranslatorMessage> ret;
-    QHash<TranslatorMessagePtr, int>::ConstIterator it = dups.constBegin(), end = dups.constEnd();
-    for (; it != end; ++it)
-        if (it.value() > 1)
-            ret.append(*it.key().ptr);
-    return ret;
-}
-
-void Translator::resolveDualEncoded()
-{
-    QHash<TranslatorMessagePtr, int> dups;
+    QSet<TranslatorMessagePtr> dups;
+    QHash<TranslatorMessagePtr, int> refs;
     for (int i = 0; i < m_messages.count();) {
         const TranslatorMessage &msg = m_messages.at(i);
-        QHash<TranslatorMessagePtr, int>::ConstIterator it = dups.constFind(msg);
-        if (it != dups.constEnd()) {
+        QHash<TranslatorMessagePtr, int>::ConstIterator it = refs.constFind(msg);
+        if (it != refs.constEnd()) {
             TranslatorMessage &omsg = m_messages[*it];
             if (omsg.isUtf8() != msg.isUtf8() && !omsg.isNonUtf8()) {
+                // Dual-encoded message
                 omsg.setUtf8(true);
                 omsg.setNonUtf8(true);
-                m_messages.removeAt(i);
-                continue;
+            } else {
+                // Duplicate
+                dups.insert(omsg);
             }
-            // Regular dupe; will complain later
+            if (!omsg.isTranslated() && msg.isTranslated())
+                omsg.setTranslations(msg.translations());
+            m_messages.removeAt(i);
+        } else {
+            refs[msg] = i;
+            ++i;
+        }
+    }
+    return dups;
+}
+
+void Translator::reportDuplicates(const QSet<TranslatorMessagePtr> &dupes,
+                                  const QString &fileName, bool verbose)
+{
+    if (!dupes.isEmpty()) {
+        if (!verbose) {
+            qWarning("Warning: dropping duplicate messages in '%s'\n(try -verbose for more info).",
+                     qPrintable(fileName));
         } else {
-            dups[msg] = i;
+            qWarning("Warning: dropping duplicate messages in '%s':", qPrintable(fileName));
+            foreach (const TranslatorMessagePtr &msg, dupes) {
+                qWarning("\n* Context: %s\n* Source: %s",
+                        qPrintable(msg->context()),
+                        qPrintable(msg->sourceText()));
+                if (!msg->comment().isEmpty())
+                    qWarning("* Comment: %s", qPrintable(msg->comment()));
+            }
+            qWarning();
         }
-        ++i;
     }
 }
 
diff --git a/tools/linguist/shared/translator.h b/tools/linguist/shared/translator.h
index 8908305..6b88b23 100644
--- a/tools/linguist/shared/translator.h
+++ b/tools/linguist/shared/translator.h
@@ -127,9 +127,10 @@ public:
     void stripNonPluralForms();
     void stripIdenticalSourceTranslations();
     void dropTranslations();
-    QList<TranslatorMessage> findDuplicates() const;
-    void resolveDualEncoded();
     void makeFileNamesAbsolute(const QDir &originalPath);
+    QSet<TranslatorMessagePtr> resolveDuplicates();
+    static void reportDuplicates(const QSet<TranslatorMessagePtr> &dupes,
+                                 const QString &fileName, bool verbose);
 
     void setCodecName(const QByteArray &name);
     QByteArray codecName() const { return m_codecName; }
diff --git a/tools/linguist/shared/translatormessage.h b/tools/linguist/shared/translatormessage.h
index 363019e..7d31925 100644
--- a/tools/linguist/shared/translatormessage.h
+++ b/tools/linguist/shared/translatormessage.h
@@ -179,6 +179,32 @@ Q_DECLARE_TYPEINFO(TranslatorMessage, Q_MOVABLE_TYPE);
 
 int qHash(const TranslatorMessage &msg);
 
+struct TranslatorMessagePtr {
+    TranslatorMessagePtr(const TranslatorMessage &tm)
+    {
+        ptr = &tm;
+    }
+
+    inline const TranslatorMessage *operator->() const
+    {
+        return ptr;
+    }
+
+    const TranslatorMessage *ptr;
+};
+
+Q_DECLARE_TYPEINFO(TranslatorMessagePtr, Q_MOVABLE_TYPE);
+
+static inline int qHash(TranslatorMessagePtr tmp)
+{
+    return qHash(*tmp.ptr);
+}
+
+static inline bool operator==(TranslatorMessagePtr tmp1, TranslatorMessagePtr tmp2)
+{
+    return *tmp1.ptr == *tmp2.ptr;
+}
+
 QT_END_NAMESPACE
 
 #endif // QT_NO_TRANSLATION
diff --git a/tools/linguist/shared/ts.cpp b/tools/linguist/shared/ts.cpp
index 8a6d365..22f2a1b 100644
--- a/tools/linguist/shared/ts.cpp
+++ b/tools/linguist/shared/ts.cpp
@@ -242,7 +242,7 @@ bool TSReader::read(Translator &translator)
     STRING(userdata);
     STRING(utf8);
     STRING(value);
-    STRING(version);
+    //STRING(version);
     STRING(yes);
 
     static const QString strextrans(QLatin1String("extra-"));
@@ -266,15 +266,13 @@ bool TSReader::read(Translator &translator)
             QString currentFile;
 
             QXmlStreamAttributes atts = attributes();
-            QString version = atts.value(strversion).toString();
+            //QString version = atts.value(strversion).toString();
             translator.setLanguageCode(atts.value(strlanguage).toString());
             translator.setSourceLanguageCode(atts.value(strsourcelanguage).toString());
             while (!atEnd()) {
                 readNext();
                 if (isEndElement()) {
                     // </TS> found, finish local loop
-                    if (version == QLatin1String("1.1"))
-                        translator.resolveDualEncoded();
                     break;
                 } else if (isWhiteSpace()) {
                     // ignore these, just whitespace