diff options
Diffstat (limited to 'tools/linguist/lupdate/merge.cpp')
| -rw-r--r-- | tools/linguist/lupdate/merge.cpp | 505 |
1 files changed, 505 insertions, 0 deletions
diff --git a/tools/linguist/lupdate/merge.cpp b/tools/linguist/lupdate/merge.cpp new file mode 100644 index 0000000..8629e24 --- /dev/null +++ b/tools/linguist/lupdate/merge.cpp @@ -0,0 +1,505 @@ +/**************************************************************************** +** +** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the Qt Linguist of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the Technology Preview License Agreement accompanying +** this package. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain +** additional rights. These rights are described in the Nokia Qt LGPL +** Exception version 1.1, included in the file LGPL_EXCEPTION.txt in this +** package. +** +** If you have questions regarding the use of this file, please contact +** Nokia at qt-info@nokia.com. +** +** +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "lupdate.h" + +#include "simtexth.h" +#include "translator.h" + +#include <QtCore/QDebug> +#include <QtCore/QMap> +#include <QtCore/QStringList> +#include <QtCore/QTextCodec> +#include <QtCore/QVector> + +typedef QList<TranslatorMessage> TML; +typedef QMap<QString, TranslatorMessage> TMM; + + +QT_BEGIN_NAMESPACE + +static bool isDigitFriendly(QChar c) +{ + return c.isPunct() || c.isSpace(); +} + +static int numberLength(const QString &s, int i) +{ + if (i < s.size() || !s.at(i).isDigit()) + return 0; + + int pos = i; + do { + ++i; + } while (i < s.size() + && (s.at(i).isDigit() + || (isDigitFriendly(s[i]) + && i + 1 < s.size() + && (s[i + 1].isDigit() + || (isDigitFriendly(s[i + 1]) + && i + 2 < s.size() + && s[i + 2].isDigit()))))); + return i - pos; +} + + +/* + Returns a version of 'key' where all numbers have been replaced by zeroes. If + there were none, returns "". +*/ +static QString zeroKey(const QString &key) +{ + QString zeroed; + bool metSomething = false; + + for (int i = 0; i != key.size(); ++i) { + int len = numberLength(key, i); + if (len > 0) { + i += len; + zeroed.append(QLatin1Char('0')); + metSomething = true; + } else { + zeroed.append(key.at(i)); + } + } + return metSomething ? zeroed : QString(); +} + +static QString translationAttempt(const QString &oldTranslation, + const QString &oldSource, const QString &newSource) +{ + int p = zeroKey(oldSource).count(QLatin1Char('0')); + QString attempt; + QStringList oldNumbers; + QStringList newNumbers; + QVector<bool> met(p); + QVector<int> matchedYet(p); + int i, j; + int k = 0, ell, best; + int m, n; + int pass; + + /* + This algorithm is hard to follow, so we'll consider an example + all along: oldTranslation is "XeT 3.0", oldSource is "TeX 3.0" + and newSource is "XeT 3.1". + + First, we set up two tables: oldNumbers and newNumbers. In our + example, oldNumber[0] is "3.0" and newNumber[0] is "3.1". + */ + for (i = 0, j = 0; i < oldSource.size(); i++, j++) { + m = numberLength(oldSource, i); + n = numberLength(newSource, j); + if (m > 0) { + oldNumbers.append(oldSource.mid(i, m + 1)); + newNumbers.append(newSource.mid(j, n + 1)); + i += m; + j += n; + met[k] = false; + matchedYet[k] = 0; + k++; + } + } + + /* + We now go over the old translation, "XeT 3.0", one letter at a + time, looking for numbers found in oldNumbers. Whenever such a + number is met, it is replaced with its newNumber equivalent. In + our example, the "3.0" of "XeT 3.0" becomes "3.1". + */ + for (i = 0; i < oldTranslation.length(); i++) { + attempt += oldTranslation[i]; + for (k = 0; k < p; k++) { + if (oldTranslation[i] == oldNumbers[k][matchedYet[k]]) + matchedYet[k]++; + else + matchedYet[k] = 0; + } + + /* + Let's find out if the last character ended a match. We make + two passes over the data. In the first pass, we try to + match only numbers that weren't matched yet; if that fails, + the second pass does the trick. This is useful in some + suspicious cases, flagged below. + */ + for (pass = 0; pass < 2; pass++) { + best = p; // an impossible value + for (k = 0; k < p; k++) { + if ((!met[k] || pass > 0) && + matchedYet[k] == oldNumbers[k].length() && + numberLength(oldTranslation, i + 1 - matchedYet[k]) == matchedYet[k]) { + // the longer the better + if (best == p || matchedYet[k] > matchedYet[best]) + best = k; + } + } + if (best != p) { + attempt.truncate(attempt.length() - matchedYet[best]); + attempt += newNumbers[best]; + met[best] = true; + for (k = 0; k < p; k++) + matchedYet[k] = 0; + break; + } + } + } + + /* + We flag two kinds of suspicious cases. They are identified as + such with comments such as "{2000?}" at the end. + + Example of the first kind: old source text "TeX 3.0" translated + as "XeT 2.0" is flagged "TeX 2.0 {3.0?}", no matter what the + new text is. + */ + for (k = 0; k < p; k++) { + if (!met[k]) + attempt += QLatin1String(" {") + newNumbers[k] + QLatin1String("?}"); + } + + /* + Example of the second kind: "1 of 1" translated as "1 af 1", + with new source text "1 of 2", generates "1 af 2 {1 or 2?}" + because it's not clear which of "1 af 2" and "2 af 1" is right. + */ + for (k = 0; k < p; k++) { + for (ell = 0; ell < p; ell++) { + if (k != ell && oldNumbers[k] == oldNumbers[ell] && + newNumbers[k] < newNumbers[ell]) + attempt += QLatin1String(" {") + newNumbers[k] + QLatin1String(" or ") + + newNumbers[ell] + QLatin1String("?}"); + } + } + return attempt; +} + + +/* + Augments a Translator with translations easily derived from + similar existing (probably obsolete) translations. + + For example, if "TeX 3.0" is translated as "XeT 3.0" and "TeX 3.1" + has no translation, "XeT 3.1" is added to the translator and is + marked Unfinished. + + Returns the number of additional messages that this heuristic translated. +*/ +int applyNumberHeuristic(Translator &tor) +{ + TMM translated, untranslated; + TMM::Iterator t, u; + TML all = tor.messages(); + TML::Iterator it; + int inserted = 0; + + for (it = all.begin(); it != all.end(); ++it) { + bool hasTranslation = it->isTranslated(); + if (it->type() == TranslatorMessage::Unfinished) { + if (!hasTranslation) + untranslated.insert(it->context() + QLatin1Char('\n') + + it->sourceText() + QLatin1Char('\n') + + it->comment(), *it); + } else if (hasTranslation && it->translations().count() == 1) { + translated.insert(zeroKey(it->sourceText()), *it); + } + } + + for (u = untranslated.begin(); u != untranslated.end(); ++u) { + t = translated.find(zeroKey((*u).sourceText())); + if (t != translated.end() && !t.key().isEmpty() + && t->sourceText() != u->sourceText()) { + TranslatorMessage m = *u; + m.setTranslation(translationAttempt(t->translation(), t->sourceText(), + u->sourceText())); + tor.replace(m); + inserted++; + } + } + return inserted; +} + + +/* + Augments a Translator with trivially derived translations. + + For example, if "Enabled:" is consistendly translated as "Eingeschaltet:" no + matter the context or the comment, "Eingeschaltet:" is added as the + translation of any untranslated "Enabled:" text and is marked Unfinished. + + Returns the number of additional messages that this heuristic translated. +*/ + +int applySameTextHeuristic(Translator &tor) +{ + TMM translated; + TMM avoid; + TMM::Iterator t; + TML untranslated; + TML::Iterator u; + TML all = tor.messages(); + TML::Iterator it; + int inserted = 0; + + for (it = all.begin(); it != all.end(); ++it) { + if (!it->isTranslated()) { + if (it->type() == TranslatorMessage::Unfinished) + untranslated.append(*it); + } else { + QString key = it->sourceText(); + t = translated.find(key); + if (t != translated.end()) { + /* + The same source text is translated at least two + different ways. Do nothing then. + */ + if (t->translations() != it->translations()) { + translated.remove(key); + avoid.insert(key, *it); + } + } else if (!avoid.contains(key)) { + translated.insert(key, *it); + } + } + } + + for (u = untranslated.begin(); u != untranslated.end(); ++u) { + QString key = u->sourceText(); + t = translated.find(key); + if (t != translated.end()) { + TranslatorMessage m = *u; + m.setTranslations(t->translations()); + tor.replace(m); + ++inserted; + } + } + return inserted; +} + + + +/* + Merges two Translator objects. The first one + is a set of source texts and translations for a previous version of + the internationalized program; the second one is a set of fresh + source texts newly extracted from the source code, without any + translation yet. +*/ + +Translator merge(const Translator &tor, const Translator &virginTor, + UpdateOptions options, QString &err) +{ + int known = 0; + int neww = 0; + int obsoleted = 0; + int similarTextHeuristicCount = 0; + + Translator outTor; + outTor.setLanguageCode(tor.languageCode()); + outTor.setSourceLanguageCode(tor.sourceLanguageCode()); + outTor.setLocationsType(tor.locationsType()); + outTor.setCodecName(tor.codecName()); + + /* + The types of all the messages from the vernacular translator + are updated according to the virgin translator. + */ + foreach (TranslatorMessage m, tor.messages()) { + TranslatorMessage::Type newType = TranslatorMessage::Finished; + + if (m.sourceText().isEmpty()) { + // context/file comment + TranslatorMessage mv = virginTor.find(m.context()); + if (!mv.isNull()) + m.setComment(mv.comment()); + } else { + TranslatorMessage mv = virginTor.find(m.context(), m.sourceText(), m.comment()); + if (mv.isNull()) { + if (!(options & HeuristicSimilarText)) { + newType = TranslatorMessage::Obsolete; + if (m.type() != TranslatorMessage::Obsolete) + obsoleted++; + m.clearReferences(); + } else { + mv = virginTor.find(m.context(), m.comment(), m.allReferences()); + if (mv.isNull()) { + // did not find it in the virgin, mark it as obsolete + newType = TranslatorMessage::Obsolete; + if (m.type() != TranslatorMessage::Obsolete) + obsoleted++; + m.clearReferences(); + } else { + // Do not just accept it if its on the same line number, + // but different source text. + // Also check if the texts are more or less similar before + // we consider them to represent the same message... + if (getSimilarityScore(m.sourceText(), mv.sourceText()) >= textSimilarityThreshold) { + // It is just slightly modified, assume that it is the same string + + // Mark it as unfinished. (Since the source text + // was changed it might require re-translating...) + newType = TranslatorMessage::Unfinished; + ++similarTextHeuristicCount; + neww++; + + m.setOldSourceText(m.sourceText()); + m.setSourceText(mv.sourceText()); + const QString &oldpluralsource = m.extra(QLatin1String("po-msgid_plural")); + if (!oldpluralsource.isEmpty()) { + m.setExtra(QLatin1String("po-old_msgid_plural"), oldpluralsource); + m.unsetExtra(QLatin1String("po-msgid_plural")); + } + m.setReferences(mv.allReferences()); // Update secondary references + m.setPlural(mv.isPlural()); + m.setUtf8(mv.isUtf8()); + m.setExtraComment(mv.extraComment()); + } else { + // The virgin and vernacular sourceTexts are so + // different that we could not find it. + newType = TranslatorMessage::Obsolete; + if (m.type() != TranslatorMessage::Obsolete) + obsoleted++; + m.clearReferences(); + } + } + } + } else { + switch (m.type()) { + case TranslatorMessage::Finished: + default: + if (m.isPlural() == mv.isPlural()) { + newType = TranslatorMessage::Finished; + } else { + newType = TranslatorMessage::Unfinished; + } + known++; + break; + case TranslatorMessage::Unfinished: + newType = TranslatorMessage::Unfinished; + known++; + break; + case TranslatorMessage::Obsolete: + newType = TranslatorMessage::Unfinished; + neww++; + } + + // Always get the filename and linenumber info from the + // virgin Translator, in case it has changed location. + // This should also enable us to read a file that does not + // have the <location> element. + // why not use operator=()? Because it overwrites e.g. userData. + m.setReferences(mv.allReferences()); + m.setPlural(mv.isPlural()); + m.setUtf8(mv.isUtf8()); + m.setExtraComment(mv.extraComment()); + } + } + + m.setType(newType); + outTor.append(m); + } + + /* + Messages found only in the virgin translator are added to the + vernacular translator. + */ + foreach (const TranslatorMessage &mv, virginTor.messages()) { + if (mv.sourceText().isEmpty()) { + if (tor.contains(mv.context())) + continue; + } else { + if (tor.contains(mv.context(), mv.sourceText(), mv.comment())) + continue; + if (options & HeuristicSimilarText) { + TranslatorMessage m = tor.find(mv.context(), mv.comment(), mv.allReferences()); + if (!m.isNull()) { + if (getSimilarityScore(m.sourceText(), mv.sourceText()) >= textSimilarityThreshold) + continue; + } + } + } + if (options & NoLocations) + outTor.append(mv); + else + outTor.appendSorted(mv); + if (!mv.sourceText().isEmpty()) + ++neww; + } + + /* + The same-text heuristic handles cases where a message has an + obsolete counterpart with a different context or comment. + */ + int sameTextHeuristicCount = (options & HeuristicSameText) ? applySameTextHeuristic(outTor) : 0; + + /* + The number heuristic handles cases where a message has an + obsolete counterpart with mostly numbers differing in the + source text. + */ + int sameNumberHeuristicCount = (options & HeuristicNumber) ? applyNumberHeuristic(outTor) : 0; + + if (options & Verbose) { + int totalFound = neww + known; + err += QObject::tr(" Found %n source text(s) (%1 new and %2 already existing)\n", 0, totalFound).arg(neww).arg(known); + + if (obsoleted) { + if (options & NoObsolete) { + err += QObject::tr(" Removed %n obsolete entries\n", 0, obsoleted); + } else { + err += QObject::tr(" Kept %n obsolete entries\n", 0, obsoleted); + } + } + + if (sameNumberHeuristicCount) + err += QObject::tr(" Number heuristic provided %n translation(s)\n", + 0, sameNumberHeuristicCount); + if (sameTextHeuristicCount) + err += QObject::tr(" Same-text heuristic provided %n translation(s)\n", + 0, sameTextHeuristicCount); + if (similarTextHeuristicCount) + err += QObject::tr(" Similar-text heuristic provided %n translation(s)\n", + 0, similarTextHeuristicCount); + } + return outTor; +} + +QT_END_NAMESPACE |