From 385bb34a9dc8e7bd6d1d3c11247862b87978d629 Mon Sep 17 00:00:00 2001 From: Oswald Buddenhagen Date: Wed, 1 Apr 2009 17:51:27 +0200 Subject: duplicate message handling improvements - lrelease will not abort on duplicates any more - lconvert now gets noisy, but only if messages within one file are duplicated (combining files with identical messages is ok) - lupdate stays silent, but will eliminate duplicates again - consolidate handling of dual-encoded and duplicate messages - for performance - instead of the format loader, now the app is responsible for calling the duplicate handler. this allows for the fine-grained control necessary for optimal performance. Task-number: 247738 --- tools/linguist/lconvert/main.cpp | 2 + tools/linguist/linguist/messagemodel.cpp | 10 ++-- tools/linguist/lrelease/main.cpp | 15 +----- tools/linguist/lupdate/main.cpp | 1 + tools/linguist/shared/qm.cpp | 1 - tools/linguist/shared/translator.cpp | 79 ++++++++++++++----------------- tools/linguist/shared/translator.h | 5 +- tools/linguist/shared/translatormessage.h | 26 ++++++++++ tools/linguist/shared/ts.cpp | 6 +-- 9 files changed, 77 insertions(+), 68 deletions(-) diff --git a/tools/linguist/lconvert/main.cpp b/tools/linguist/lconvert/main.cpp index 2842bc7..9ccc60e 100644 --- a/tools/linguist/lconvert/main.cpp +++ b/tools/linguist/lconvert/main.cpp @@ -210,6 +210,7 @@ int main(int argc, char *argv[]) qWarning() << qPrintable(cd.error()); return 2; } + Translator::reportDuplicates(tr.resolveDuplicates(), inFiles[0].name, verbose); for (int i = 1; i < inFiles.size(); ++i) { Translator tr2; @@ -217,6 +218,7 @@ int main(int argc, char *argv[]) qWarning() << qPrintable(cd.error()); return 2; } + Translator::reportDuplicates(tr2.resolveDuplicates(), inFiles[i].name, verbose); for (int j = 0; j < tr2.messageCount(); ++j) tr.replaceSorted(tr2.message(j)); } diff --git a/tools/linguist/linguist/messagemodel.cpp b/tools/linguist/linguist/messagemodel.cpp index b049ab9..7a8063b 100644 --- a/tools/linguist/linguist/messagemodel.cpp +++ b/tools/linguist/linguist/messagemodel.cpp @@ -209,19 +209,19 @@ bool DataModel::load(const QString &fileName, bool *langGuessed, QWidget *parent return false; } - QList dupes = tor.findDuplicates(); + QSet dupes = tor.resolveDuplicates(); if (!dupes.isEmpty()) { QString err = tr("Duplicate messages found in '%1':").arg(Qt::escape(fileName)); int numdups = 0; - foreach (const TranslatorMessage &msg, dupes) { + foreach (const TranslatorMessagePtr &msg, dupes) { if (++numdups >= 5) { err += tr("

[more duplicates omitted]"); break; } err += tr("

* Context: %1
* Source: %2") - .arg(Qt::escape(msg.context()), Qt::escape(msg.sourceText())); - if (!msg.comment().isEmpty()) - err += tr("
* Comment: %3").arg(Qt::escape(msg.comment())); + .arg(Qt::escape(msg->context()), Qt::escape(msg->sourceText())); + if (!msg->comment().isEmpty()) + err += tr("
* Comment: %3").arg(Qt::escape(msg->comment())); } QMessageBox::warning(parent, QObject::tr("Qt Linguist"), err); } diff --git a/tools/linguist/lrelease/main.cpp b/tools/linguist/lrelease/main.cpp index f1fdb3a..3bcc998 100644 --- a/tools/linguist/lrelease/main.cpp +++ b/tools/linguist/lrelease/main.cpp @@ -94,19 +94,6 @@ static bool loadTsFile(Translator &tor, const QString &tsFileName, bool /* verbo } else { if (!cd.errors().isEmpty()) printOut(cd.error()); - const QList dupes = tor.findDuplicates(); - if (!dupes.isEmpty()) { - qWarning("lrelease error: duplicate messages found in '%s':", - qPrintable(tsFileName)); - foreach (const TranslatorMessage &msg, dupes) { - qWarning("\n* Context: %s\n* Source: %s", - qPrintable(msg.context()), - qPrintable(msg.sourceText())); - if (!msg.comment().isEmpty()) - qWarning("\n* Comment: %s", qPrintable(msg.comment())); - } - ok = false; - } } return ok; } @@ -115,6 +102,8 @@ static bool releaseTranslator(Translator &tor, const QString &qmFileName, bool verbose, bool ignoreUnfinished, bool removeIdentical, TranslatorSaveMode mode) { + Translator::reportDuplicates(tor.resolveDuplicates(), qmFileName, verbose); + if (verbose) printOut(QCoreApplication::tr( "Updating '%1'...\n").arg(qmFileName)); if (removeIdentical) { diff --git a/tools/linguist/lupdate/main.cpp b/tools/linguist/lupdate/main.cpp index 5869838..b537b6e 100644 --- a/tools/linguist/lupdate/main.cpp +++ b/tools/linguist/lupdate/main.cpp @@ -146,6 +146,7 @@ static void updateTsFiles(const Translator &fetchedTor, const QStringList &tsFil *fail = true; continue; } + tor.resolveDuplicates(); cd.clearErrors(); if (!codecForTr.isEmpty() && codecForTr != tor.codecName()) qWarning("lupdate warning: Codec for tr() '%s' disagrees with " diff --git a/tools/linguist/shared/qm.cpp b/tools/linguist/shared/qm.cpp index d19b519..5563ac5 100644 --- a/tools/linguist/shared/qm.cpp +++ b/tools/linguist/shared/qm.cpp @@ -659,7 +659,6 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd) msg.setComment(comment); translator.append(msg); } - translator.resolveDualEncoded(); return ok; } diff --git a/tools/linguist/shared/translator.cpp b/tools/linguist/shared/translator.cpp index 9ac1dd8..312bb71 100644 --- a/tools/linguist/shared/translator.cpp +++ b/tools/linguist/shared/translator.cpp @@ -416,59 +416,52 @@ void Translator::dropTranslations() } } -struct TranslatorMessagePtr { - TranslatorMessagePtr(const TranslatorMessage &tm) - { - ptr = &tm; - } - - const TranslatorMessage *ptr; -}; - -Q_DECLARE_TYPEINFO(TranslatorMessagePtr, Q_MOVABLE_TYPE); - -static inline int qHash(TranslatorMessagePtr tmp) +QSet Translator::resolveDuplicates() { - return qHash(*tmp.ptr); -} - -static inline bool operator==(TranslatorMessagePtr tmp1, TranslatorMessagePtr tmp2) -{ - return *tmp1.ptr == *tmp2.ptr; -} - -QList Translator::findDuplicates() const -{ - QHash dups; - foreach (const TranslatorMessage &msg, m_messages) - dups[msg]++; - QList ret; - QHash::ConstIterator it = dups.constBegin(), end = dups.constEnd(); - for (; it != end; ++it) - if (it.value() > 1) - ret.append(*it.key().ptr); - return ret; -} - -void Translator::resolveDualEncoded() -{ - QHash dups; + QSet dups; + QHash refs; for (int i = 0; i < m_messages.count();) { const TranslatorMessage &msg = m_messages.at(i); - QHash::ConstIterator it = dups.constFind(msg); - if (it != dups.constEnd()) { + QHash::ConstIterator it = refs.constFind(msg); + if (it != refs.constEnd()) { TranslatorMessage &omsg = m_messages[*it]; if (omsg.isUtf8() != msg.isUtf8() && !omsg.isNonUtf8()) { + // Dual-encoded message omsg.setUtf8(true); omsg.setNonUtf8(true); - m_messages.removeAt(i); - continue; + } else { + // Duplicate + dups.insert(omsg); } - // Regular dupe; will complain later + if (!omsg.isTranslated() && msg.isTranslated()) + omsg.setTranslations(msg.translations()); + m_messages.removeAt(i); + } else { + refs[msg] = i; + ++i; + } + } + return dups; +} + +void Translator::reportDuplicates(const QSet &dupes, + const QString &fileName, bool verbose) +{ + if (!dupes.isEmpty()) { + if (!verbose) { + qWarning("Warning: dropping duplicate messages in '%s'\n(try -verbose for more info).", + qPrintable(fileName)); } else { - dups[msg] = i; + qWarning("Warning: dropping duplicate messages in '%s':", qPrintable(fileName)); + foreach (const TranslatorMessagePtr &msg, dupes) { + qWarning("\n* Context: %s\n* Source: %s", + qPrintable(msg->context()), + qPrintable(msg->sourceText())); + if (!msg->comment().isEmpty()) + qWarning("* Comment: %s", qPrintable(msg->comment())); + } + qWarning(); } - ++i; } } diff --git a/tools/linguist/shared/translator.h b/tools/linguist/shared/translator.h index 8908305..6b88b23 100644 --- a/tools/linguist/shared/translator.h +++ b/tools/linguist/shared/translator.h @@ -127,9 +127,10 @@ public: void stripNonPluralForms(); void stripIdenticalSourceTranslations(); void dropTranslations(); - QList findDuplicates() const; - void resolveDualEncoded(); void makeFileNamesAbsolute(const QDir &originalPath); + QSet resolveDuplicates(); + static void reportDuplicates(const QSet &dupes, + const QString &fileName, bool verbose); void setCodecName(const QByteArray &name); QByteArray codecName() const { return m_codecName; } diff --git a/tools/linguist/shared/translatormessage.h b/tools/linguist/shared/translatormessage.h index 363019e..7d31925 100644 --- a/tools/linguist/shared/translatormessage.h +++ b/tools/linguist/shared/translatormessage.h @@ -179,6 +179,32 @@ Q_DECLARE_TYPEINFO(TranslatorMessage, Q_MOVABLE_TYPE); int qHash(const TranslatorMessage &msg); +struct TranslatorMessagePtr { + TranslatorMessagePtr(const TranslatorMessage &tm) + { + ptr = &tm; + } + + inline const TranslatorMessage *operator->() const + { + return ptr; + } + + const TranslatorMessage *ptr; +}; + +Q_DECLARE_TYPEINFO(TranslatorMessagePtr, Q_MOVABLE_TYPE); + +static inline int qHash(TranslatorMessagePtr tmp) +{ + return qHash(*tmp.ptr); +} + +static inline bool operator==(TranslatorMessagePtr tmp1, TranslatorMessagePtr tmp2) +{ + return *tmp1.ptr == *tmp2.ptr; +} + QT_END_NAMESPACE #endif // QT_NO_TRANSLATION diff --git a/tools/linguist/shared/ts.cpp b/tools/linguist/shared/ts.cpp index 8a6d365..22f2a1b 100644 --- a/tools/linguist/shared/ts.cpp +++ b/tools/linguist/shared/ts.cpp @@ -242,7 +242,7 @@ bool TSReader::read(Translator &translator) STRING(userdata); STRING(utf8); STRING(value); - STRING(version); + //STRING(version); STRING(yes); static const QString strextrans(QLatin1String("extra-")); @@ -266,15 +266,13 @@ bool TSReader::read(Translator &translator) QString currentFile; QXmlStreamAttributes atts = attributes(); - QString version = atts.value(strversion).toString(); + //QString version = atts.value(strversion).toString(); translator.setLanguageCode(atts.value(strlanguage).toString()); translator.setSourceLanguageCode(atts.value(strsourcelanguage).toString()); while (!atEnd()) { readNext(); if (isEndElement()) { // found, finish local loop - if (version == QLatin1String("1.1")) - translator.resolveDualEncoded(); break; } else if (isWhiteSpace()) { // ignore these, just whitespace -- cgit v0.12