Add support for comments and meta-data in the lupdate QtScript frontend

This brings the support more in line with the C++ frontend.
No support for magic TRANSLATOR comment yet.
Add a proper test for the QtScript parser.
Support for qtTrId() and friends coming in a separate commit.

Previously the lexer just skipped comments. Now the contents
of each comment is retained and passed to a "comment processor"
interface. The parser implements the interface and performs
logic very similar to the C++ parser to extract data from
the comment.

Task-number: QTBUG-11774
Reviewed-by: Oswald Buddenhagen

5 files changed, 565 insertions, 22 deletions

diff --git a/tests/auto/linguist/lupdate/testdata/good/parsejs/main.js b/tests/auto/linguist/lupdate/testdata/good/parsejs/main.js
new file mode 100644
index 0000000..edd7529
--- /dev/null
+++ b/tests/auto/linguist/lupdate/testdata/good/parsejs/main.js
@@ -0,0 +1,83 @@
+qsTr("One");
+qsTranslate("FooContext", "Two");
+
+var greeting_strings = [
+    QT_TR_NOOP("Hello"),
+    QT_TRANSLATE_NOOP("FooContext", "Goodbye")
+];
+
+qsTr("One", "not the same one");
+
+//: My first comment.
+qsTr("See comment");
+
+//: My second comment.
+qsTranslate("BarContext", "See other comment");
+
+//: My third comment
+//: spans two lines.
+qsTr("The comment explains it all");
+
+//: My fourth comment
+//: spans a whopping
+//: three lines.
+qsTranslate("BazContext", "It should be clear by now");
+
+/*: C-style comment. */
+qsTr("I love C++");
+
+/*: Another C-style comment. */
+qsTranslate("FooContext", "I really love C++");
+
+/*: C-style comment, followed by */
+/*: another one. */
+qsTr("Qt is the best");
+
+/*: Another C-style comment, followed by */
+/*: yet another one. */
+qsTranslate("BarContext", "Qt is the very best");
+
+// This comment doesn't have any effect.
+qsTr("The comment had no effect");
+
+// This comment doesn't have any effect either.
+qsTranslate("BazContext", "The comment had no effect, really");
+
+/* This C-style comment doesn't have any effect. */
+qsTr("No comment to your comment");
+
+/* This C-style comment doesn't have any effect either. */
+qsTranslate("FooContext", "I refuse to comment on that");
+
+//= id_foo
+qsTr("This string has an identifier");
+
+//= id_bar
+qsTranslate("BarContext", "This string also has an identifier");
+
+//~ loc-blank False
+qsTr("This string has meta-data");
+
+//~ loc-layout_id foo_dialog
+qsTranslate("BazContext", "This string also has meta-data");
+
+// This comment is to be ignored.
+//: This is a comment for the translator.
+//= id_baz
+//~ foo 123
+//~ magic-stuff This means something special.
+qsTr("This string has a lot of information");
+
+// This comment is also to be ignored.
+//: This is another comment for the translator.
+//= id_babar
+//~ foo-bar Important stuff
+//~ needle-in-haystack Found
+//~ overflow True
+qsTranslate("FooContext", "This string has even more information");
+
+qsTr("This string has disambiguation", "Disambiguation");
+
+qsTranslate("BarContext", "This string also has disambiguation", "Another disambiguation");
+
+qsTr("This string contains plurals", "", 10);
diff --git a/tests/auto/linguist/lupdate/testdata/good/parsejs/project.pro b/tests/auto/linguist/lupdate/testdata/good/parsejs/project.pro
new file mode 100644
index 0000000..d549039
--- /dev/null
+++ b/tests/auto/linguist/lupdate/testdata/good/parsejs/project.pro
@@ -0,0 +1,3 @@
+SOURCES += main.js
+
+TRANSLATIONS = project.ts
diff --git a/tests/auto/linguist/lupdate/testdata/good/parsejs/project.ts.result b/tests/auto/linguist/lupdate/testdata/good/parsejs/project.ts.result
new file mode 100644
index 0000000..d2016de
--- /dev/null
+++ b/tests/auto/linguist/lupdate/testdata/good/parsejs/project.ts.result
@@ -0,0 +1,168 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE TS>
+<TS version="2.0">
+<context>
+    <name>BarContext</name>
+    <message>
+        <location filename="main.js" line="15"/>
+        <source>See other comment</source>
+        <extracomment>My second comment.</extracomment>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="38"/>
+        <source>Qt is the very best</source>
+        <extracomment>Another C-style comment, followed by yet another one.</extracomment>
+        <translation type="unfinished"></translation>
+    </message>
+    <message id="id_bar">
+        <location filename="main.js" line="56"/>
+        <source>This string also has an identifier</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="81"/>
+        <source>This string also has disambiguation</source>
+        <comment>Another disambiguation</comment>
+        <translation type="unfinished"></translation>
+    </message>
+</context>
+<context>
+    <name>BazContext</name>
+    <message>
+        <location filename="main.js" line="24"/>
+        <source>It should be clear by now</source>
+        <extracomment>My fourth comment spans a whopping three lines.</extracomment>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="44"/>
+        <source>The comment had no effect, really</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="62"/>
+        <source>This string also has meta-data</source>
+        <translation type="unfinished"></translation>
+        <extra-loc-layout_id>foo_dialog</extra-loc-layout_id>
+    </message>
+</context>
+<context>
+    <name>FooContext</name>
+    <message>
+        <location filename="main.js" line="2"/>
+        <source>Two</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="6"/>
+        <source>Goodbye</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="30"/>
+        <source>I really love C++</source>
+        <extracomment>Another C-style comment.</extracomment>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="50"/>
+        <source>I refuse to comment on that</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message id="id_babar">
+        <location filename="main.js" line="77"/>
+        <source>This string has even more information</source>
+        <extracomment>This is another comment for the translator.</extracomment>
+        <translation type="unfinished"></translation>
+        <extra-needle-in-haystack>Found</extra-needle-in-haystack>
+        <extra-overflow>True</extra-overflow>
+        <extra-foo-bar>Important stuff</extra-foo-bar>
+    </message>
+</context>
+<context>
+    <name>main</name>
+    <message>
+        <location filename="main.js" line="1"/>
+        <source>One</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="5"/>
+        <source>Hello</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="9"/>
+        <source>One</source>
+        <comment>not the same one</comment>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="12"/>
+        <source>See comment</source>
+        <extracomment>My first comment.</extracomment>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="19"/>
+        <source>The comment explains it all</source>
+        <extracomment>My third comment spans two lines.</extracomment>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="27"/>
+        <source>I love C++</source>
+        <extracomment>C-style comment.</extracomment>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="34"/>
+        <source>Qt is the best</source>
+        <extracomment>C-style comment, followed by another one.</extracomment>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="41"/>
+        <source>The comment had no effect</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="47"/>
+        <source>No comment to your comment</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message id="id_foo">
+        <location filename="main.js" line="53"/>
+        <source>This string has an identifier</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.js" line="59"/>
+        <source>This string has meta-data</source>
+        <translation type="unfinished"></translation>
+        <extra-loc-blank>False</extra-loc-blank>
+    </message>
+    <message id="id_baz">
+        <location filename="main.js" line="69"/>
+        <source>This string has a lot of information</source>
+        <extracomment>This is a comment for the translator.</extracomment>
+        <translation type="unfinished"></translation>
+        <extra-foo>123</extra-foo>
+        <extra-magic-stuff>This means something special.</extra-magic-stuff>
+    </message>
+    <message>
+        <location filename="main.js" line="79"/>
+        <source>This string has disambiguation</source>
+        <comment>Disambiguation</comment>
+        <translation type="unfinished"></translation>
+    </message>
+    <message numerus="yes">
+        <location filename="main.js" line="83"/>
+        <source>This string contains plurals</source>
+        <translation type="unfinished">
+            <numerusform></numerusform>
+        </translation>
+    </message>
+</context>
+</TS>
diff --git a/tools/linguist/lupdate/qscript.cpp b/tools/linguist/lupdate/qscript.cpp
index 08670ac..dc885a3 100644
--- a/tools/linguist/lupdate/qscript.cpp
+++ b/tools/linguist/lupdate/qscript.cpp
@@ -770,13 +770,16 @@ const int QScriptGrammar::action_check [] = {
 
 static void recordMessage(
     Translator *tor, const QString &context, const QString &text, const QString &comment,
-    const QString &extracomment, bool plural, const QString &fileName, int lineNo)
+    const QString &extracomment, const QString &msgid, const TranslatorMessage::ExtraData &extra,
+    bool plural, const QString &fileName, int lineNo)
 {
     TranslatorMessage msg(
         context, text, comment, QString(),
         fileName, lineNo, QStringList(),
         TranslatorMessage::Unfinished, plural);
     msg.setExtraComment(extracomment.simplified());
+    msg.setId(msgid);
+    msg.setExtras(extra);
     tor->extend(msg);
 }
 
@@ -784,10 +787,17 @@ static void recordMessage(
 namespace QScript
 {
 
+class CommentProcessor
+{
+public:
+    virtual ~CommentProcessor() {}
+    virtual void processComment(const QChar *chars, int length) = 0;
+};
+
 class Lexer
 {
 public:
-    Lexer();
+    Lexer(CommentProcessor *);
     ~Lexer();
 
     void setCode(const QString &c, const QString &fileName, int lineno);
@@ -927,6 +937,8 @@ private:
 
     void syncProhibitAutomaticSemicolon();
 
+    void processComment(const QChar *, int);
+
     const QChar *code;
     uint length;
     int yycolumn;
@@ -953,6 +965,8 @@ private:
     ParenthesesState parenthesesState;
     int parenthesesCount;
     bool prohibitAutomaticSemicolon;
+
+    CommentProcessor *commentProcessor;
 };
 
 } // namespace QScript
@@ -1029,7 +1043,7 @@ double integerFromString(const char *buf, int size, int radix)
 
 } // namespace QScript
 
-QScript::Lexer::Lexer()
+QScript::Lexer::Lexer(QScript::CommentProcessor *proc)
     :
       yylineno(0),
       size8(128), size16(128), restrKeyword(false),
@@ -1040,7 +1054,8 @@ QScript::Lexer::Lexer()
       err(NoError),
       check_reserved(true),
       parenthesesState(IgnoreParentheses),
-      prohibitAutomaticSemicolon(false)
+      prohibitAutomaticSemicolon(false),
+      commentProcessor(proc)
 {
     // allocate space for read buffers
     buffer8 = new char[size8];
@@ -1407,10 +1422,12 @@ int QScript::Lexer::lex()
             } else if (current == '/' && next1 == '/') {
                 recordStartPos();
                 shift(1);
+                Q_ASSERT(pos16 == 0);
                 state = InSingleLineComment;
             } else if (current == '/' && next1 == '*') {
                 recordStartPos();
                 shift(1);
+                Q_ASSERT(pos16 == 0);
                 state = InMultiLineComment;
             } else if (current == 0) {
                 syncProhibitAutomaticSemicolon();
@@ -1550,9 +1567,12 @@ int QScript::Lexer::lex()
             break;
         case InSingleLineComment:
             if (isLineTerminator()) {
+                record16(current); // include newline
+                processComment(buffer16, pos16);
                 shiftWindowsLineBreak();
                 yylineno++;
                 yycolumn = 0;
+                pos16 = 0;
                 terminator = true;
                 bol = true;
                 if (restrKeyword) {
@@ -1562,6 +1582,8 @@ int QScript::Lexer::lex()
                     state = Start;
             } else if (current == 0) {
                 setDone(Eof);
+            } else {
+                record16(current);
             }
             break;
         case InMultiLineComment:
@@ -1573,8 +1595,12 @@ int QScript::Lexer::lex()
                 shiftWindowsLineBreak();
                 yylineno++;
             } else if (current == '*' && next1 == '/') {
+                processComment(buffer16, pos16);
+                pos16 = 0;
                 state = Start;
                 shift(1);
+            } else {
+                record16(current);
             }
             break;
         case InIdentifier:
@@ -2036,10 +2062,15 @@ void QScript::Lexer::syncProhibitAutomaticSemicolon()
     }
 }
 
+void QScript::Lexer::processComment(const QChar *chars, int length)
+{
+    commentProcessor->processComment(chars, length);
+}
+
 
 class Translator;
 
-class QScriptParser: protected QScriptGrammar
+class QScriptParser: protected QScriptGrammar, public QScript::CommentProcessor
 {
 public:
     QVariant val;
@@ -2079,6 +2110,8 @@ protected:
 
     std::ostream &yyMsg(int line = 0);
 
+    virtual void processComment(const QChar *, int);
+
 protected:
     int tos;
     int stack_size;
@@ -2091,6 +2124,10 @@ protected:
 
 private:
     QScript::Lexer *lexer;
+    QString extracomment;
+    QString msgid;
+    QString sourcetext;
+    TranslatorMessage::ExtraData extra;
 };
 
 inline void QScriptParser::reallocateStack()
@@ -2229,6 +2266,8 @@ case 8: {
 case 66: {
     QString name = sym(1).toString();
     if ((name == QLatin1String("qsTranslate")) || (name == QLatin1String("QT_TRANSLATE_NOOP"))) {
+        if (!sourcetext.isEmpty())
+            yyMsg(identLineNo) << "//% cannot be used with " << qPrintable(name) << "(). Ignoring\n";
         QVariantList args = sym(2).toList();
         if (args.size() < 2) {
             yyMsg(identLineNo) << qPrintable(name) << "() requires at least two arguments.\n";
@@ -2240,13 +2279,18 @@ case 66: {
                 QString context = args.at(0).toString();
                 QString text = args.at(1).toString();
                 QString comment = args.value(2).toString();
-                QString extracomment;
                 bool plural = (args.size() > 4);
                 recordMessage(translator, context, text, comment, extracomment,
-                              plural, fileName(), identLineNo);
+                              msgid, extra, plural, fileName(), identLineNo);
             }
         }
+        sourcetext.clear();
+        extracomment.clear();
+        msgid.clear();
+        extra.clear();
     } else if ((name == QLatin1String("qsTr")) || (name == QLatin1String("QT_TR_NOOP"))) {
+        if (!sourcetext.isEmpty())
+            yyMsg(identLineNo) << "//% cannot be used with " << qPrintable(name) << "(). Ignoring\n";
         QVariantList args = sym(2).toList();
         if (args.size() < 1) {
             yyMsg(identLineNo) << qPrintable(name) << "() requires at least one argument.\n";
@@ -2257,12 +2301,15 @@ case 66: {
                 QString context = QFileInfo(fileName()).baseName();
                 QString text = args.at(0).toString();
                 QString comment = args.value(1).toString();
-                QString extracomment;
                 bool plural = (args.size() > 2);
                 recordMessage(translator, context, text, comment, extracomment,
-                              plural, fileName(), identLineNo);
+                              msgid, extra, plural, fileName(), identLineNo);
             }
         }
+        sourcetext.clear();
+        extracomment.clear();
+        msgid.clear();
+        extra.clear();
     }
 } break;
 
@@ -2289,6 +2336,44 @@ case 94: {
         sym(1) = QVariant();
 } break;
 
+    case 171:
+
+    case 172:
+
+    case 173:
+
+    case 174:
+
+    case 175:
+
+    case 176:
+
+    case 177:
+
+    case 178:
+
+    case 179:
+
+    case 180:
+
+    case 181:
+
+    case 182:
+
+    case 183:
+
+    case 184:
+
+    case 185:
+    if (!sourcetext.isEmpty() || !extracomment.isEmpty() || !msgid.isEmpty() || !extra.isEmpty()) {
+        yyMsg() << "Discarding unconsumed meta data\n";
+        sourcetext.clear();
+        extracomment.clear();
+        msgid.clear();
+        extra.clear();
+    }
+    break;
+
           } // switch
 
           state_stack [tos] = nt_action (act, lhs [r] - TERMINAL_COUNT);
@@ -2372,6 +2457,58 @@ std::ostream &QScriptParser::yyMsg(int line)
     return std::cerr << qPrintable(fileName()) << ':' << (line ? line : lexer->startLineNo()) << ": ";
 }
 
+void QScriptParser::processComment(const QChar *chars, int length)
+{
+    if (!length)
+        return;
+    // Try to match the logic of the C++ parser.
+    if (*chars == QLatin1Char(':') && chars[1].isSpace()) {
+        extracomment += QString(chars+2, length-2);
+    } else if (*chars == QLatin1Char('=') && chars[1].isSpace()) {
+        msgid = QString(chars+2, length-2).simplified();
+    } else if (*chars == QLatin1Char('~') && chars[1].isSpace()) {
+        QString text = QString(chars+2, length-2).trimmed();
+        int k = text.indexOf(QLatin1Char(' '));
+        if (k > -1)
+            extra.insert(text.left(k), text.mid(k + 1).trimmed());
+    } else if (*chars == QLatin1Char('%') && chars[1].isSpace()) {
+        sourcetext.reserve(sourcetext.length() + length-2);
+        ushort *ptr = (ushort *)sourcetext.data() + sourcetext.length();
+        int p = 2, c;
+        forever {
+            if (p >= length)
+                break;
+            c = chars[p++].unicode();
+            if (isspace(c))
+                continue;
+            if (c != '"') {
+                yyMsg() << "Unexpected character in meta string\n";
+                break;
+            }
+            forever {
+                if (p >= length) {
+                  whoops:
+                    yyMsg() << "Unterminated meta string\n";
+                    break;
+                }
+                c = chars[p++].unicode();
+                if (c == '"')
+                    break;
+                if (c == '\\') {
+                    if (p >= length)
+                        goto whoops;
+                    c = chars[p++].unicode();
+                    if (c == '\n')
+                        goto whoops;
+                    *ptr++ = '\\';
+                }
+                *ptr++ = c;
+            }
+        }
+        sourcetext.resize(ptr - (ushort *)sourcetext.data());
+    }
+}
+
 
 bool loadQScript(Translator &translator, const QString &filename, ConversionData &cd)
 {
@@ -2391,9 +2528,9 @@ bool loadQScript(Translator &translator, const QString &filename, ConversionData
     ts.setAutoDetectUnicode(true);
 
     QString code = ts.readAll();
-    QScript::Lexer lexer;
-    lexer.setCode(code, filename, /*lineNumber=*/1);
     QScriptParser parser;
+    QScript::Lexer lexer(&parser);
+    lexer.setCode(code, filename, /*lineNumber=*/1);
     parser.setLexer(&lexer);
     if (!parser.parse(&translator)) {
         std::cerr << qPrintable(filename) << ':' << parser.errorLineNumber() << ": "
diff --git a/tools/linguist/lupdate/qscript.g b/tools/linguist/lupdate/qscript.g
index 2f82a55..200b641 100644
--- a/tools/linguist/lupdate/qscript.g
+++ b/tools/linguist/lupdate/qscript.g
@@ -101,13 +101,16 @@ QT_BEGIN_NAMESPACE
 
 static void recordMessage(
     Translator *tor, const QString &context, const QString &text, const QString &comment,
-    const QString &extracomment, bool plural, const QString &fileName, int lineNo)
+    const QString &extracomment, const QString &msgid, const TranslatorMessage::ExtraData &extra,
+    bool plural, const QString &fileName, int lineNo)
 {
     TranslatorMessage msg(
         context, text, comment, QString(),
         fileName, lineNo, QStringList(),
         TranslatorMessage::Unfinished, plural);
     msg.setExtraComment(extracomment.simplified());
+    msg.setId(msgid);
+    msg.setExtras(extra);
     tor->extend(msg);
 }
 
@@ -115,10 +118,17 @@ static void recordMessage(
 namespace QScript
 {
 
+class CommentProcessor
+{
+public:
+    virtual ~CommentProcessor() {}
+    virtual void processComment(const QChar *chars, int length) = 0;
+};
+
 class Lexer
 {
 public:
-    Lexer();
+    Lexer(CommentProcessor *);
     ~Lexer();
 
     void setCode(const QString &c, const QString &fileName, int lineno);
@@ -258,6 +268,8 @@ private:
 
     void syncProhibitAutomaticSemicolon();
 
+    void processComment(const QChar *, int);
+
     const QChar *code;
     uint length;
     int yycolumn;
@@ -284,6 +296,8 @@ private:
     ParenthesesState parenthesesState;
     int parenthesesCount;
     bool prohibitAutomaticSemicolon;
+
+    CommentProcessor *commentProcessor;
 };
 
 } // namespace QScript
@@ -360,7 +374,7 @@ double integerFromString(const char *buf, int size, int radix)
 
 } // namespace QScript
 
-QScript::Lexer::Lexer()
+QScript::Lexer::Lexer(QScript::CommentProcessor *proc)
     :
       yylineno(0),
       size8(128), size16(128), restrKeyword(false),
@@ -371,7 +385,8 @@ QScript::Lexer::Lexer()
       err(NoError),
       check_reserved(true),
       parenthesesState(IgnoreParentheses),
-      prohibitAutomaticSemicolon(false)
+      prohibitAutomaticSemicolon(false),
+      commentProcessor(proc)
 {
     // allocate space for read buffers
     buffer8 = new char[size8];
@@ -738,10 +753,12 @@ int QScript::Lexer::lex()
             } else if (current == '/' && next1 == '/') {
                 recordStartPos();
                 shift(1);
+                Q_ASSERT(pos16 == 0);
                 state = InSingleLineComment;
             } else if (current == '/' && next1 == '*') {
                 recordStartPos();
                 shift(1);
+                Q_ASSERT(pos16 == 0);
                 state = InMultiLineComment;
             } else if (current == 0) {
                 syncProhibitAutomaticSemicolon();
@@ -881,9 +898,12 @@ int QScript::Lexer::lex()
             break;
         case InSingleLineComment:
             if (isLineTerminator()) {
+                record16(current); // include newline
+                processComment(buffer16, pos16);
                 shiftWindowsLineBreak();
                 yylineno++;
                 yycolumn = 0;
+                pos16 = 0;
                 terminator = true;
                 bol = true;
                 if (restrKeyword) {
@@ -893,6 +913,8 @@ int QScript::Lexer::lex()
                     state = Start;
             } else if (current == 0) {
                 setDone(Eof);
+            } else {
+                record16(current);
             }
             break;
         case InMultiLineComment:
@@ -904,8 +926,12 @@ int QScript::Lexer::lex()
                 shiftWindowsLineBreak();
                 yylineno++;
             } else if (current == '*' && next1 == '/') {
+                processComment(buffer16, pos16);
+                pos16 = 0;
                 state = Start;
                 shift(1);
+            } else {
+                record16(current);
             }
             break;
         case InIdentifier:
@@ -1367,10 +1393,15 @@ void QScript::Lexer::syncProhibitAutomaticSemicolon()
     }
 }
 
+void QScript::Lexer::processComment(const QChar *chars, int length)
+{
+    commentProcessor->processComment(chars, length);
+}
+
 
 class Translator;
 
-class QScriptParser: protected $table
+class QScriptParser: protected $table, public QScript::CommentProcessor
 {
 public:
     QVariant val;
@@ -1410,6 +1441,8 @@ protected:
 
     std::ostream &yyMsg(int line = 0);
 
+    virtual void processComment(const QChar *, int);
+
 protected:
     int tos;
     int stack_size;
@@ -1422,6 +1455,10 @@ protected:
 
 private:
     QScript::Lexer *lexer;
+    QString extracomment;
+    QString msgid;
+    QString sourcetext;
+    TranslatorMessage::ExtraData extra;
 };
 
 inline void QScriptParser::reallocateStack()
@@ -1645,6 +1682,8 @@ CallExpression: MemberExpression Arguments ;
 case $rule_number: {
     QString name = sym(1).toString();
     if ((name == QLatin1String("qsTranslate")) || (name == QLatin1String("QT_TRANSLATE_NOOP"))) {
+        if (!sourcetext.isEmpty())
+            yyMsg(identLineNo) << "//% cannot be used with " << qPrintable(name) << "(). Ignoring\n";
         QVariantList args = sym(2).toList();
         if (args.size() < 2) {
             yyMsg(identLineNo) << qPrintable(name) << "() requires at least two arguments.\n";
@@ -1656,13 +1695,18 @@ case $rule_number: {
                 QString context = args.at(0).toString();
                 QString text = args.at(1).toString();
                 QString comment = args.value(2).toString();
-                QString extracomment;
                 bool plural = (args.size() > 4);
                 recordMessage(translator, context, text, comment, extracomment,
-                              plural, fileName(), identLineNo);
+                              msgid, extra, plural, fileName(), identLineNo);
             }
         }
+        sourcetext.clear();
+        extracomment.clear();
+        msgid.clear();
+        extra.clear();
     } else if ((name == QLatin1String("qsTr")) || (name == QLatin1String("QT_TR_NOOP"))) {
+        if (!sourcetext.isEmpty())
+            yyMsg(identLineNo) << "//% cannot be used with " << qPrintable(name) << "(). Ignoring\n";
         QVariantList args = sym(2).toList();
         if (args.size() < 1) {
             yyMsg(identLineNo) << qPrintable(name) << "() requires at least one argument.\n";
@@ -1673,12 +1717,15 @@ case $rule_number: {
                 QString context = QFileInfo(fileName()).baseName();
                 QString text = args.at(0).toString();
                 QString comment = args.value(1).toString();
-                QString extracomment;
                 bool plural = (args.size() > 2);
                 recordMessage(translator, context, text, comment, extracomment,
-                              plural, fileName(), identLineNo);
+                              msgid, extra, plural, fileName(), identLineNo);
             }
         }
+        sourcetext.clear();
+        extracomment.clear();
+        msgid.clear();
+        extra.clear();
     }
 } break;
 ./
@@ -1824,20 +1871,73 @@ ExpressionNotInOpt: ;
 ExpressionNotInOpt: ExpressionNotIn ;
 
 Statement: Block ;
+/.
+    case $rule_number:
+./
 Statement: VariableStatement ;
+/.
+    case $rule_number:
+./
 Statement: EmptyStatement ;
+/.
+    case $rule_number:
+./
 Statement: ExpressionStatement ;
+/.
+    case $rule_number:
+./
 Statement: IfStatement ;
+/.
+    case $rule_number:
+./
 Statement: IterationStatement ;
+/.
+    case $rule_number:
+./
 Statement: ContinueStatement ;
+/.
+    case $rule_number:
+./
 Statement: BreakStatement ;
+/.
+    case $rule_number:
+./
 Statement: ReturnStatement ;
+/.
+    case $rule_number:
+./
 Statement: WithStatement ;
+/.
+    case $rule_number:
+./
 Statement: LabelledStatement ;
+/.
+    case $rule_number:
+./
 Statement: SwitchStatement ;
+/.
+    case $rule_number:
+./
 Statement: ThrowStatement ;
+/.
+    case $rule_number:
+./
 Statement: TryStatement ;
+/.
+    case $rule_number:
+./
 Statement: DebuggerStatement ;
+/.
+    case $rule_number:
+    if (!sourcetext.isEmpty() || !extracomment.isEmpty() || !msgid.isEmpty() || !extra.isEmpty()) {
+        yyMsg() << "Discarding unconsumed meta data\n";
+        sourcetext.clear();
+        extracomment.clear();
+        msgid.clear();
+        extra.clear();
+    }
+    break;
+./
 
 Block: T_LBRACE StatementListOpt T_RBRACE ;
 StatementList: Statement ;
@@ -2004,6 +2104,58 @@ std::ostream &QScriptParser::yyMsg(int line)
     return std::cerr << qPrintable(fileName()) << ':' << (line ? line : lexer->startLineNo()) << ": ";
 }
 
+void QScriptParser::processComment(const QChar *chars, int length)
+{
+    if (!length)
+        return;
+    // Try to match the logic of the C++ parser.
+    if (*chars == QLatin1Char(':') && chars[1].isSpace()) {
+        extracomment += QString(chars+2, length-2);
+    } else if (*chars == QLatin1Char('=') && chars[1].isSpace()) {
+        msgid = QString(chars+2, length-2).simplified();
+    } else if (*chars == QLatin1Char('~') && chars[1].isSpace()) {
+        QString text = QString(chars+2, length-2).trimmed();
+        int k = text.indexOf(QLatin1Char(' '));
+        if (k > -1)
+            extra.insert(text.left(k), text.mid(k + 1).trimmed());
+    } else if (*chars == QLatin1Char('%') && chars[1].isSpace()) {
+        sourcetext.reserve(sourcetext.length() + length-2);
+        ushort *ptr = (ushort *)sourcetext.data() + sourcetext.length();
+        int p = 2, c;
+        forever {
+            if (p >= length)
+                break;
+            c = chars[p++].unicode();
+            if (isspace(c))
+                continue;
+            if (c != '"') {
+                yyMsg() << "Unexpected character in meta string\n";
+                break;
+            }
+            forever {
+                if (p >= length) {
+                  whoops:
+                    yyMsg() << "Unterminated meta string\n";
+                    break;
+                }
+                c = chars[p++].unicode();
+                if (c == '"')
+                    break;
+                if (c == '\\') {
+                    if (p >= length)
+                        goto whoops;
+                    c = chars[p++].unicode();
+                    if (c == '\n')
+                        goto whoops;
+                    *ptr++ = '\\';
+                }
+                *ptr++ = c;
+            }
+        }
+        sourcetext.resize(ptr - (ushort *)sourcetext.data());
+    }
+}
+
 
 bool loadQScript(Translator &translator, const QString &filename, ConversionData &cd)
 {
@@ -2023,9 +2175,9 @@ bool loadQScript(Translator &translator, const QString &filename, ConversionData
     ts.setAutoDetectUnicode(true);
 
     QString code = ts.readAll();
-    QScript::Lexer lexer;
-    lexer.setCode(code, filename, /*lineNumber=*/1);
     QScriptParser parser;
+    QScript::Lexer lexer(&parser);
+    lexer.setCode(code, filename, /*lineNumber=*/1);
     parser.setLexer(&lexer);
     if (!parser.parse(&translator)) {
         std::cerr << qPrintable(filename) << ':' << parser.errorLineNumber() << ": "