summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorOswald Buddenhagen <oswald.buddenhagen@nokia.com>2009-03-26 22:13:30 (GMT)
committerOswald Buddenhagen <oswald.buddenhagen@nokia.com>2009-03-27 11:08:34 (GMT)
commitd3be9fa7c981430c48de0a65938f3a2bd636bfcc (patch)
tree36ba96ebbab08a5cd770b9f9d731d37e7be63313
parent32798196d81439fa77b34425b95eb47556aebc8d (diff)
downloadQt-d3be9fa7c981430c48de0a65938f3a2bd636bfcc.zip
Qt-d3be9fa7c981430c48de0a65938f3a2bd636bfcc.tar.gz
Qt-d3be9fa7c981430c48de0a65938f3a2bd636bfcc.tar.bz2
properly deal with messages which appear in multiple encodings
in ts 1.1 and qm files, messages appear in their native encoding. that means that a message can appear multiple times - once in utf8 and once in the codecForTr. however, in ts 2.0 files, everything is utf8 and messages can have a utf8 flag for the later transformation into qm. unfortunately, there was no flag to mark that the message is needed in *both* encodings, and the respective case was completely ignored when reading ts 1.1 and qm files (causing error messages). Task-number: 249022 AutoTest: 322690
-rw-r--r--tools/linguist/shared/qm.cpp291
-rw-r--r--tools/linguist/shared/translator.cpp26
-rw-r--r--tools/linguist/shared/translator.h1
-rw-r--r--tools/linguist/shared/translatormessage.cpp4
-rw-r--r--tools/linguist/shared/translatormessage.h3
-rw-r--r--tools/linguist/shared/ts.cpp194
6 files changed, 299 insertions, 220 deletions
diff --git a/tools/linguist/shared/qm.cpp b/tools/linguist/shared/qm.cpp
index 4627b5d..8a3658c 100644
--- a/tools/linguist/shared/qm.cpp
+++ b/tools/linguist/shared/qm.cpp
@@ -103,6 +103,43 @@ static uint elfHash(const QByteArray &ba)
return h;
}
+class ByteTranslatorMessage
+{
+public:
+ ByteTranslatorMessage(
+ const QByteArray &context,
+ const QByteArray &sourceText,
+ const QByteArray &comment,
+ const QStringList &translations) :
+ m_context(context),
+ m_sourcetext(sourceText),
+ m_comment(comment),
+ m_translations(translations)
+ {}
+ const QByteArray &context() const { return m_context; }
+ const QByteArray &sourceText() const { return m_sourcetext; }
+ const QByteArray &comment() const { return m_comment; }
+ const QStringList &translations() const { return m_translations; }
+ bool operator<(const ByteTranslatorMessage& m) const;
+
+private:
+ QByteArray m_context;
+ QByteArray m_sourcetext;
+ QByteArray m_comment;
+ QStringList m_translations;
+};
+
+Q_DECLARE_TYPEINFO(ByteTranslatorMessage, Q_MOVABLE_TYPE);
+
+bool ByteTranslatorMessage::operator<(const ByteTranslatorMessage& m) const
+{
+ if (m_context != m.m_context)
+ return m_context < m.m_context;
+ if (m_sourcetext != m.m_sourcetext)
+ return m_sourcetext < m.m_sourcetext;
+ return m_comment < m.m_comment;
+}
+
class Releaser
{
public:
@@ -133,27 +170,12 @@ public:
m_codec = QTextCodec::codecForName(codecName);
}
- TranslatorMessage findMessage(const QString &context,
- const QString &sourceText, const QString &comment,
- const QString &fileName = QString(), int lineNumber = -1) const;
-
bool save(QIODevice *iod);
- void insert(const TranslatorMessage &);
- void remove(const TranslatorMessage &);
-
- bool contains(const QString &context, const QString &sourceText,
- const QString & comment) const;
-
- bool contains(const QString &context, const QString &comment,
- const QString &fileName, int lineNumber) const;
+ void insert(const TranslatorMessage &msg, bool forceComment);
void squeeze(TranslatorSaveMode mode);
- QList<TranslatorMessage> messages() const;
-
- bool isEmpty() const;
-
void setNumerusRules(const QByteArray &rules);
private:
@@ -163,18 +185,20 @@ private:
// on turn should be the same as passed to the actual tr(...) calls
QByteArray originalBytes(const QString &str, bool isUtf8) const;
- Prefix commonPrefix(const TranslatorMessage &m1, const TranslatorMessage &m2) const;
+ void insertInternal(const TranslatorMessage &message, bool forceComment, bool isUtf8);
- uint msgHash(const TranslatorMessage &msg) const;
+ Prefix commonPrefix(const ByteTranslatorMessage &m1, const ByteTranslatorMessage &m2) const;
- void writeMessage(const TranslatorMessage & msg, QDataStream & stream,
+ uint msgHash(const ByteTranslatorMessage &msg) const;
+
+ void writeMessage(const ByteTranslatorMessage & msg, QDataStream & stream,
TranslatorSaveMode strip, Prefix prefix) const;
// for squeezed but non-file data, this is what needs to be deleted
QByteArray m_messageArray;
QByteArray m_offsetArray;
QByteArray m_contextArray;
- QMap<TranslatorMessage, void *> m_messages;
+ QMap<ByteTranslatorMessage, void *> m_messages;
QByteArray m_numerusRules;
// Used to reproduce the original bytes
@@ -193,12 +217,12 @@ QByteArray Releaser::originalBytes(const QString &str, bool isUtf8) const
return m_codec ? m_codec->fromUnicode(str) : str.toLatin1();
}
-uint Releaser::msgHash(const TranslatorMessage &msg) const
+uint Releaser::msgHash(const ByteTranslatorMessage &msg) const
{
- return elfHash(originalBytes(msg.sourceText() + msg.comment(), msg.isUtf8()));
+ return elfHash(msg.sourceText() + msg.comment());
}
-Prefix Releaser::commonPrefix(const TranslatorMessage &m1, const TranslatorMessage &m2) const
+Prefix Releaser::commonPrefix(const ByteTranslatorMessage &m1, const ByteTranslatorMessage &m2) const
{
if (msgHash(m1) != msgHash(m2))
return NoPrefix;
@@ -211,7 +235,7 @@ Prefix Releaser::commonPrefix(const TranslatorMessage &m1, const TranslatorMessa
return HashContextSourceTextComment;
}
-void Releaser::writeMessage(const TranslatorMessage & msg, QDataStream & stream,
+void Releaser::writeMessage(const ByteTranslatorMessage &msg, QDataStream &stream,
TranslatorSaveMode mode, Prefix prefix) const
{
for (int i = 0; i < msg.translations().count(); ++i) {
@@ -228,14 +252,14 @@ void Releaser::writeMessage(const TranslatorMessage & msg, QDataStream & stream,
switch (prefix) {
default:
case HashContextSourceTextComment:
- stream << quint8(Tag_Comment) << originalBytes(msg.comment(), msg.isUtf8());
+ stream << quint8(Tag_Comment) << msg.comment();
// fall through
case HashContextSourceText:
- stream << quint8(Tag_SourceText) << originalBytes(msg.sourceText(), msg.isUtf8());
+ stream << quint8(Tag_SourceText) << msg.sourceText();
// fall through
case HashContext:
- stream << quint8(Tag_Context) << originalBytes(msg.context(), msg.isUtf8());
- ;
+ stream << quint8(Tag_Context) << msg.context();
+ break;
}
stream << quint8(Tag_End);
@@ -275,7 +299,7 @@ void Releaser::squeeze(TranslatorSaveMode mode)
if (m_messages.isEmpty() && mode == SaveEverything)
return;
- QMap<TranslatorMessage, void *> messages = m_messages;
+ QMap<ByteTranslatorMessage, void *> messages = m_messages;
// re-build contents
m_messageArray.clear();
@@ -286,7 +310,7 @@ void Releaser::squeeze(TranslatorSaveMode mode)
QMap<Offset, void *> offsets;
QDataStream ms(&m_messageArray, QIODevice::WriteOnly);
- QMap<TranslatorMessage, void *>::const_iterator it, next;
+ QMap<ByteTranslatorMessage, void *>::const_iterator it, next;
int cpPrev = 0, cpNext = 0;
for (it = messages.constBegin(); it != messages.constEnd(); ++it) {
cpPrev = cpNext;
@@ -310,7 +334,7 @@ void Releaser::squeeze(TranslatorSaveMode mode)
}
if (mode == SaveStripped) {
- QMap<QString, int> contextSet;
+ QMap<QByteArray, int> contextSet;
for (it = messages.constBegin(); it != messages.constEnd(); ++it)
++contextSet[it.key().context()];
@@ -322,10 +346,10 @@ void Releaser::squeeze(TranslatorSaveMode mode)
else
hTableSize = (contextSet.size() < 10000) ? 15013 : 3 * contextSet.size() / 2;
- QMultiMap<int, QString> hashMap;
- QMap<QString, int>::const_iterator c;
+ QMultiMap<int, QByteArray> hashMap;
+ QMap<QByteArray, int>::const_iterator c;
for (c = contextSet.constBegin(); c != contextSet.constEnd(); ++c)
- hashMap.insert(elfHash(originalBytes(c.key(), false /*FIXME*/)) % hTableSize, c.key());
+ hashMap.insert(elfHash(c.key()) % hTableSize, c.key());
/*
The contexts found in this translator are stored in a hash
@@ -360,16 +384,14 @@ void Releaser::squeeze(TranslatorSaveMode mode)
t << quint16(0); // the entry at offset 0 cannot be used
uint upto = 2;
- QMap<int, QString>::const_iterator entry = hashMap.constBegin();
+ QMap<int, QByteArray>::const_iterator entry = hashMap.constBegin();
while (entry != hashMap.constEnd()) {
int i = entry.key();
hTable[i] = quint16(upto >> 1);
do {
- QString context = entry.value();
- QByteArray ba = context.toUtf8();
- const char *con = ba.data();
- uint len = uint(qstrlen(con));
+ const char *con = entry.value().constData();
+ uint len = uint(entry.value().length());
len = qMin(len, 255u);
t << quint8(len);
t.writeRawData(con, len);
@@ -394,68 +416,28 @@ void Releaser::squeeze(TranslatorSaveMode mode)
}
}
-bool Releaser::contains(const QString &context, const QString &sourceText,
- const QString &comment) const
-{
- return !findMessage(context, sourceText, comment).translation().isNull();
-}
-
-bool Releaser::contains(const QString &context, const QString &comment,
- const QString &fileName, int lineNumber) const
-{
- return !findMessage(context, QString(), comment, fileName, lineNumber).isNull();
-}
-
-void Releaser::insert(const TranslatorMessage &message)
-{
- m_messages.insert(message, 0);
-}
-
-void Releaser::remove(const TranslatorMessage &message)
-{
- m_messages.remove(message);
-}
-
-
-TranslatorMessage Releaser::findMessage(const QString &context,
- const QString &sourceText, const QString &comment,
- const QString &fileName, int lineNumber) const
+void Releaser::insertInternal(const TranslatorMessage &message, bool forceComment, bool isUtf8)
{
- if (m_messages.isEmpty())
- return TranslatorMessage();
-
- QMap<TranslatorMessage, void *>::const_iterator it;
-
- // Either we want to find an item that matches context, sourcetext
- // (and optionally comment) Or we want to find an item that
- // matches context, filename, linenumber (and optionally comment)
- TranslatorMessage msg(context, sourceText, comment, QString(), fileName, lineNumber);
- it = m_messages.constFind(msg);
- if (it != m_messages.constEnd())
- return it.key();
-
- if (!comment.isEmpty()) {
- it = m_messages.constFind(TranslatorMessage(context, sourceText, QString(), QString(), fileName, lineNumber));
- if (it != m_messages.constEnd())
- return it.key();
+ ByteTranslatorMessage bmsg(originalBytes(message.context(), isUtf8),
+ originalBytes(message.sourceText(), isUtf8),
+ originalBytes(message.comment(), isUtf8),
+ message.translations());
+ if (!forceComment) {
+ ByteTranslatorMessage bmsg2(
+ bmsg.context(), bmsg.sourceText(), QByteArray(""), bmsg.translations());
+ if (!m_messages.contains(bmsg2)) {
+ m_messages.insert(bmsg2, 0);
+ return;
+ }
}
-
- it = m_messages.constFind(TranslatorMessage(context, QString(), comment, QString(), fileName, lineNumber));
- if (it != m_messages.constEnd())
- return it.key();
- if (comment.isEmpty())
- return TranslatorMessage();
-
- it = m_messages.constFind(TranslatorMessage(context, QString(), QString(), QString(), fileName, lineNumber));
- if (it != m_messages.constEnd())
- return it.key();
- return TranslatorMessage();
+ m_messages.insert(bmsg, 0);
}
-bool Releaser::isEmpty() const
+void Releaser::insert(const TranslatorMessage &message, bool forceComment)
{
- return m_messageArray.isEmpty() && m_offsetArray.isEmpty()
- && m_contextArray.isEmpty() && m_messages.isEmpty();
+ insertInternal(message, forceComment, message.isUtf8());
+ if (message.isUtf8() && message.isNonUtf8())
+ insertInternal(message, forceComment, false);
}
void Releaser::setNumerusRules(const QByteArray &rules)
@@ -463,11 +445,6 @@ void Releaser::setNumerusRules(const QByteArray &rules)
m_numerusRules = rules;
}
-QList<TranslatorMessage> Releaser::messages() const
-{
- return m_messages.keys();
-}
-
static quint8 read8(const uchar *data)
{
return *data;
@@ -478,6 +455,31 @@ static quint32 read32(const uchar *data)
return (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | (data[3]);
}
+static void fromBytes(const char *str, int len, QTextCodec *codec, QTextCodec *utf8Codec,
+ QString *out, QString *utf8Out,
+ bool *isSystem, bool *isUtf8, bool *needs8Bit)
+{
+ for (int i = 0; i < len; ++i)
+ if (str[i] & 0x80) {
+ if (utf8Codec) {
+ QTextCodec::ConverterState cvtState;
+ *utf8Out = utf8Codec->toUnicode(str, len, &cvtState);
+ *isUtf8 = !cvtState.invalidChars;
+ }
+ QTextCodec::ConverterState cvtState;
+ *out = codec->toUnicode(str, len, &cvtState);
+ *isSystem = !cvtState.invalidChars;
+ *needs8Bit = true;
+ return;
+ }
+ *out = QString::fromLatin1(str, len);
+ *isSystem = true;
+ if (utf8Codec) {
+ *utf8Out = *out;
+ *isUtf8 = true;
+ }
+ *needs8Bit = false;
+}
bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
{
@@ -543,10 +545,19 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
size_t numItems = offsetLength / (2 * sizeof(quint32));
//qDebug() << "NUMITEMS: " << numItems;
- TranslatorMessage msg;
-
// FIXME: that's just a guess, the original locale data is lost...
QTextCodec *codec = QTextCodec::codecForLocale();
+ QTextCodec *utf8Codec = 0;
+ if (codec->name() != "UTF-8")
+ utf8Codec = QTextCodec::codecForName("UTF-8");
+
+ QString context, contextUtf8;
+ bool contextIsSystem, contextIsUtf8, contextNeeds8Bit;
+ QString sourcetext, sourcetextUtf8;
+ bool sourcetextIsSystem, sourcetextIsUtf8, sourcetextNeeds8Bit;
+ QString comment, commentUtf8;
+ bool commentIsSystem, commentIsUtf8, commentNeeds8Bit;
+ QStringList translations;
for (const uchar *start = offsetArray; start != offsetArray + (numItems << 3); start += 8) {
//quint32 hash = read32(start);
@@ -575,7 +586,7 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
}
str.replace(QChar(Translator::InternalVariantSeparator),
QChar(Translator::DefaultVariantSeparator));
- msg.appendTranslation(str);
+ translations << str;
m += len;
break;
}
@@ -588,7 +599,9 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
m += 4;
//qDebug() << "SOURCE LEN: " << len;
//qDebug() << "SOURCE: " << QByteArray((const char*)m, len);
- msg.setSourceText(codec->toUnicode(QByteArray((const char*)m, len)));
+ fromBytes((const char*)m, len, codec, utf8Codec,
+ &sourcetext, &sourcetextUtf8,
+ &sourcetextIsSystem, &sourcetextIsUtf8, &sourcetextNeeds8Bit);
m += len;
break;
}
@@ -597,7 +610,9 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
m += 4;
//qDebug() << "CONTEXT LEN: " << len;
//qDebug() << "CONTEXT: " << QByteArray((const char*)m, len);
- msg.setContext(codec->toUnicode(QByteArray((const char*)m, len)));
+ fromBytes((const char*)m, len, codec, utf8Codec,
+ &context, &contextUtf8,
+ &contextIsSystem, &contextIsUtf8, &contextNeeds8Bit);
m += len;
break;
}
@@ -606,7 +621,9 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
m += 4;
//qDebug() << "COMMENT LEN: " << len;
//qDebug() << "COMMENT: " << QByteArray((const char*)m, len);
- msg.setComment(codec->toUnicode(QByteArray((const char*)m, len)));
+ fromBytes((const char*)m, len, codec, utf8Codec,
+ &comment, &commentUtf8,
+ &commentIsSystem, &commentIsUtf8, &commentNeeds8Bit);
m += len;
break;
}
@@ -616,11 +633,33 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
}
}
end:;
+ TranslatorMessage msg;
msg.setType(TranslatorMessage::Finished);
+ msg.setTranslations(translations);
+ translations.clear();
+ if (contextNeeds8Bit || sourcetextNeeds8Bit || commentNeeds8Bit) {
+ if (utf8Codec && contextIsUtf8 && sourcetextIsUtf8 && commentIsUtf8) {
+ // The message is utf-8, but file is not.
+ msg.setUtf8(true);
+ msg.setContext(contextUtf8);
+ msg.setSourceText(sourcetextUtf8);
+ msg.setComment(commentUtf8);
+ translator.append(msg);
+ continue;
+ }
+ if (!(contextIsSystem && sourcetextIsSystem && commentIsSystem)) {
+ cd.appendError(QLatin1String(
+ "Cannot read file with current system character codec"));
+ return false;
+ }
+ // The message is 8-bit in the file's encoding (utf-8 or not).
+ }
+ msg.setContext(context);
+ msg.setSourceText(sourcetext);
+ msg.setComment(comment);
translator.append(msg);
- //qDebug() << "\nHASH:" << hash << msg.sourceText() << msg.context();
- msg.setTranslations(QStringList());
}
+ translator.resolveDualEncoded();
return ok;
}
@@ -657,30 +696,16 @@ static bool saveQM(const Translator &translator, QIODevice &dev, ConversionData
} else {
++finished;
}
- QString context = msg.context();
- QString sourceText = msg.sourceText();
- QString comment = msg.comment();
- QStringList translations = msg.translations();
-
- /*
- Drop the comment in (context, sourceText, comment),
- unless the context is empty,
- unless (context, sourceText, "") already exists or
- unless we already dropped the comment of (context,
- sourceText, comment0).
- */
- if (comment.isEmpty()
- || context.isEmpty()
- || translator.contains(context, sourceText, QString())
- || !releaser.findMessage(context, sourceText, QString()).translation()
- .isNull() ) {
- releaser.insert(msg);
- } else {
- TranslatorMessage tm(context, sourceText, QString(),
- QString(), QString(), -1, translations);
- //filename and lineNumbers will be ignored from now.
- releaser.insert(tm);
- }
+ // Drop the comment in (context, sourceText, comment),
+ // unless the context is empty,
+ // unless (context, sourceText, "") already exists or
+ // unless we already dropped the comment of (context,
+ // sourceText, comment0).
+ bool forceComment =
+ msg.comment().isEmpty()
+ || msg.context().isEmpty()
+ || translator.contains(msg.context(), msg.sourceText(), QString());
+ releaser.insert(msg, forceComment);
}
}
diff --git a/tools/linguist/shared/translator.cpp b/tools/linguist/shared/translator.cpp
index 32404a5..3721204 100644
--- a/tools/linguist/shared/translator.cpp
+++ b/tools/linguist/shared/translator.cpp
@@ -114,6 +114,10 @@ void Translator::extend(const TranslatorMessage &msg)
cmt.append(msg.extraComment());
emsg.setExtraComment(cmt);
}
+ if (msg.isUtf8() != emsg.isUtf8()) {
+ emsg.setUtf8(true);
+ emsg.setNonUtf8(true);
+ }
}
}
@@ -425,6 +429,28 @@ QList<TranslatorMessage> Translator::findDuplicates() const
return ret;
}
+void Translator::resolveDualEncoded()
+{
+ QHash<TranslatorMessage, int> dups;
+ for (int i = 0; i < m_messages.count();) {
+ const TranslatorMessage &msg = m_messages.at(i);
+ QHash<TranslatorMessage, int>::ConstIterator it = dups.constFind(msg);
+ if (it != dups.constEnd()) {
+ TranslatorMessage &omsg = m_messages[*it];
+ if (omsg.isUtf8() != msg.isUtf8() && !omsg.isNonUtf8()) {
+ omsg.setUtf8(true);
+ omsg.setNonUtf8(true);
+ m_messages.removeAt(i);
+ continue;
+ }
+ // Regular dupe; will complain later
+ } else {
+ dups[msg] = i;
+ }
+ ++i;
+ }
+}
+
// Used by lupdate to be able to search using absolute paths during merging
void Translator::makeFileNamesAbsolute(const QDir &originalPath)
{
diff --git a/tools/linguist/shared/translator.h b/tools/linguist/shared/translator.h
index bbe7654..ba719ec 100644
--- a/tools/linguist/shared/translator.h
+++ b/tools/linguist/shared/translator.h
@@ -130,6 +130,7 @@ public:
void stripIdenticalSourceTranslations();
void dropTranslations();
QList<TranslatorMessage> findDuplicates() const;
+ void resolveDualEncoded();
void makeFileNamesAbsolute(const QDir &originalPath);
void setCodecName(const QByteArray &name);
diff --git a/tools/linguist/shared/translatormessage.cpp b/tools/linguist/shared/translatormessage.cpp
index ab4301f..afe66fe 100644
--- a/tools/linguist/shared/translatormessage.cpp
+++ b/tools/linguist/shared/translatormessage.cpp
@@ -54,7 +54,7 @@
QT_BEGIN_NAMESPACE
TranslatorMessage::TranslatorMessage()
- : m_lineNumber(-1), m_type(Unfinished), m_utf8(false), m_plural(false)
+ : m_lineNumber(-1), m_type(Unfinished), m_utf8(false), m_nonUtf8(false), m_plural(false)
{
}
@@ -66,7 +66,7 @@ TranslatorMessage::TranslatorMessage(const QString &context,
: m_context(context), m_sourcetext(sourceText), m_comment(comment),
m_userData(userData),
m_translations(translations), m_fileName(fileName), m_lineNumber(lineNumber),
- m_type(type), m_utf8(false), m_plural(plural)
+ m_type(type), m_utf8(false), m_nonUtf8(false), m_plural(plural)
{
}
diff --git a/tools/linguist/shared/translatormessage.h b/tools/linguist/shared/translatormessage.h
index fa37ff5..2818e28 100644
--- a/tools/linguist/shared/translatormessage.h
+++ b/tools/linguist/shared/translatormessage.h
@@ -136,6 +136,8 @@ public:
void setType(Type t) { m_type = t; }
bool isUtf8() const { return m_utf8; } // codecForTr override
void setUtf8(bool on) { m_utf8 = on; }
+ bool isNonUtf8() const { return m_nonUtf8; } // codecForTr override
+ void setNonUtf8(bool on) { m_nonUtf8 = on; }
bool isPlural() const { return m_plural; }
void setPlural(bool isplural) { m_plural = isplural; }
@@ -169,6 +171,7 @@ private:
Type m_type;
bool m_utf8;
+ bool m_nonUtf8;
bool m_plural;
};
diff --git a/tools/linguist/shared/ts.cpp b/tools/linguist/shared/ts.cpp
index 3af9e59..8a6d365 100644
--- a/tools/linguist/shared/ts.cpp
+++ b/tools/linguist/shared/ts.cpp
@@ -212,6 +212,7 @@ QString TSReader::readTransContents()
bool TSReader::read(Translator &translator)
{
+ STRING(both);
STRING(byte);
STRING(comment);
STRING(context);
@@ -265,13 +266,15 @@ bool TSReader::read(Translator &translator)
QString currentFile;
QXmlStreamAttributes atts = attributes();
- //QString version = atts.value(strversion).toString();
+ QString version = atts.value(strversion).toString();
translator.setLanguageCode(atts.value(strlanguage).toString());
translator.setSourceLanguageCode(atts.value(strsourcelanguage).toString());
while (!atEnd()) {
readNext();
if (isEndElement()) {
// </TS> found, finish local loop
+ if (version == QLatin1String("1.1"))
+ translator.resolveDualEncoded();
break;
} else if (isWhiteSpace()) {
// ignore these, just whitespace
@@ -311,7 +314,9 @@ bool TSReader::read(Translator &translator)
msg.setContext(context);
msg.setType(TranslatorMessage::Finished);
msg.setPlural(attributes().value(strnumerus) == stryes);
- msg.setUtf8(attributes().value(strutf8) == strtrue
+ const QStringRef &utf8Attr = attributes().value(strutf8);
+ msg.setNonUtf8(utf8Attr == strboth);
+ msg.setUtf8(msg.isNonUtf8() || utf8Attr == strtrue
|| attributes().value(strencoding) == strUtf8);
while (!atEnd()) {
readNext();
@@ -594,107 +599,126 @@ bool saveTS(const Translator &translator, QIODevice &dev, ConversionData &cd, in
foreach (const TranslatorMessage &msg, messageOrder[context]) {
//msg.dump();
- t << " <message";
- if (!msg.id().isEmpty())
- t << " id=\"" << msg.id() << "\"";
- if (format == 11 && !trIsUtf8 && msg.isUtf8())
- t << " encoding=\"UTF-8\"";
- if (format == 20 && !trIsUtf8 && msg.isUtf8())
- t << " utf8=\"true\"";
- if (msg.isPlural())
- t << " numerus=\"yes\"";
- t << ">\n";
- if (translator.locationsType() != Translator::NoLocations) {
- QString cfile = currentFile;
- bool first = true;
- foreach (const TranslatorMessage::Reference &ref, msg.allReferences()) {
- QString fn = cd.m_targetDir.relativeFilePath(ref.fileName())
- .replace(QLatin1Char('\\'),QLatin1Char('/'));
- int ln = ref.lineNumber();
- QString ld;
- if (translator.locationsType() == Translator::RelativeLocations) {
- if (ln != -1) {
- int dlt = ln - currentLine[fn];
- if (dlt >= 0)
- ld.append(QLatin1Char('+'));
- ld.append(QString::number(dlt));
- currentLine[fn] = ln;
+ bool isUtf8 = msg.isUtf8();
+ bool second = false;
+ forever {
+
+ t << " <message";
+ if (!msg.id().isEmpty())
+ t << " id=\"" << msg.id() << "\"";
+ if (!trIsUtf8) {
+ if (format == 11) {
+ if (isUtf8)
+ t << " encoding=\"UTF-8\"";
+ } else {
+ if (msg.isUtf8()) {
+ if (msg.isNonUtf8())
+ t << " utf8=\"both\"";
+ else
+ t << " utf8=\"true\"";
}
+ }
+ }
+ if (msg.isPlural())
+ t << " numerus=\"yes\"";
+ t << ">\n";
+ if (translator.locationsType() != Translator::NoLocations) {
+ QString cfile = currentFile;
+ bool first = true;
+ foreach (const TranslatorMessage::Reference &ref, msg.allReferences()) {
+ QString fn = cd.m_targetDir.relativeFilePath(ref.fileName())
+ .replace(QLatin1Char('\\'),QLatin1Char('/'));
+ int ln = ref.lineNumber();
+ QString ld;
+ if (translator.locationsType() == Translator::RelativeLocations) {
+ if (ln != -1) {
+ int dlt = ln - currentLine[fn];
+ if (dlt >= 0)
+ ld.append(QLatin1Char('+'));
+ ld.append(QString::number(dlt));
+ currentLine[fn] = ln;
+ }
- if (fn != cfile) {
- if (first)
- currentFile = fn;
- cfile = fn;
+ if (fn != cfile) {
+ if (first)
+ currentFile = fn;
+ cfile = fn;
+ } else {
+ fn.clear();
+ }
+ first = false;
} else {
- fn.clear();
+ if (ln != -1)
+ ld = QString::number(ln);
}
- first = false;
- } else {
- if (ln != -1)
- ld = QString::number(ln);
+ t << " <location";
+ if (!fn.isEmpty())
+ t << " filename=\"" << fn << "\"";
+ if (!ld.isEmpty())
+ t << " line=\"" << ld << "\"";
+ t << "/>\n";
}
- t << " <location";
- if (!fn.isEmpty())
- t << " filename=\"" << fn << "\"";
- if (!ld.isEmpty())
- t << " line=\"" << ld << "\"";
- t << "/>\n";
}
- }
- t << " <source>"
- << evilBytes(msg.sourceText(), msg.isUtf8(), format, codecName)
- << "</source>\n";
+ t << " <source>"
+ << evilBytes(msg.sourceText(), isUtf8, format, codecName)
+ << "</source>\n";
- if (format != 11 && !msg.oldSourceText().isEmpty())
- t << " <oldsource>" << protect(msg.oldSourceText()) << "</oldsource>\n";
+ if (format != 11 && !msg.oldSourceText().isEmpty())
+ t << " <oldsource>" << protect(msg.oldSourceText()) << "</oldsource>\n";
- if (!msg.comment().isEmpty()) {
- t << " <comment>"
- << evilBytes(msg.comment(), msg.isUtf8(), format, codecName)
- << "</comment>\n";
- }
+ if (!msg.comment().isEmpty()) {
+ t << " <comment>"
+ << evilBytes(msg.comment(), isUtf8, format, codecName)
+ << "</comment>\n";
+ }
- if (format != 11) {
+ if (format != 11) {
- if (!msg.oldComment().isEmpty())
- t << " <oldcomment>" << protect(msg.oldComment()) << "</oldcomment>\n";
+ if (!msg.oldComment().isEmpty())
+ t << " <oldcomment>" << protect(msg.oldComment()) << "</oldcomment>\n";
- if (!msg.extraComment().isEmpty())
- t << " <extracomment>" << protect(msg.extraComment())
- << "</extracomment>\n";
+ if (!msg.extraComment().isEmpty())
+ t << " <extracomment>" << protect(msg.extraComment())
+ << "</extracomment>\n";
- if (!msg.translatorComment().isEmpty())
- t << " <translatorcomment>" << protect(msg.translatorComment())
- << "</translatorcomment>\n";
+ if (!msg.translatorComment().isEmpty())
+ t << " <translatorcomment>" << protect(msg.translatorComment())
+ << "</translatorcomment>\n";
- }
+ }
- t << " <translation";
- if (msg.type() == TranslatorMessage::Unfinished)
- t << " type=\"unfinished\"";
- else if (msg.type() == TranslatorMessage::Obsolete)
- t << " type=\"obsolete\"";
- if (msg.isPlural()) {
- t << ">";
- QStringList translns = translator.normalizedTranslations(msg, cd, &result);
- for (int j = 0; j < qMax(1, translns.count()); ++j) {
- t << "\n <numerusform";
- writeVariants(t, " ", translns[j]);
- t << "</numerusform>";
+ t << " <translation";
+ if (msg.type() == TranslatorMessage::Unfinished)
+ t << " type=\"unfinished\"";
+ else if (msg.type() == TranslatorMessage::Obsolete)
+ t << " type=\"obsolete\"";
+ if (msg.isPlural()) {
+ t << ">";
+ QStringList translns = translator.normalizedTranslations(msg, cd, &result);
+ for (int j = 0; j < qMax(1, translns.count()); ++j) {
+ t << "\n <numerusform";
+ writeVariants(t, " ", translns[j]);
+ t << "</numerusform>";
+ }
+ t << "\n ";
+ } else {
+ writeVariants(t, " ", msg.translation());
}
- t << "\n ";
- } else {
- writeVariants(t, " ", msg.translation());
- }
- t << "</translation>\n";
+ t << "</translation>\n";
+
+ if (format != 11)
+ writeExtras(t, " ", msg.extras(), drops);
- if (format != 11)
- writeExtras(t, " ", msg.extras(), drops);
+ if (!msg.userData().isEmpty())
+ t << " <userdata>" << msg.userData() << "</userdata>\n";
+ t << " </message>\n";
- if (!msg.userData().isEmpty())
- t << " <userdata>" << msg.userData() << "</userdata>\n";
- t << " </message>\n";
+ if (format != 11 || second || !msg.isUtf8() || !msg.isNonUtf8())
+ break;
+ isUtf8 = false;
+ second = true;
+ }
}
t << "</context>\n";
}