Added missing qmlcodemarker.cpp and qmlcodemarker.h files.

Added code to remove pragmas before parsing and add them to the output.
The extractPragmas() function was copied into both qmlcodeparser.cpp and
qmlcodemarker.cpp from src/declarative/qml/qdeclarativescriptparser.cpp,
and was modified to return either nothing (for the API parser) or a
list of removed pragmas (for the code marker).

7 files changed, 523 insertions, 25 deletions

diff --git a/tools/qdoc3/jscodemarker.cpp b/tools/qdoc3/jscodemarker.cpp
index 8b6a31d..84a28c6 100644
--- a/tools/qdoc3/jscodemarker.cpp
+++ b/tools/qdoc3/jscodemarker.cpp
@@ -75,7 +75,10 @@ bool JsCodeMarker::recognizeCode(const QString &code)
     QDeclarativeJS::Lexer lexer(&engine);
     QDeclarativeJS::Parser parser(&engine);
     QDeclarativeJS::NodePool m_nodePool("<JsCodeMarker::recognizeCode>", &engine);
-    lexer.setCode(code, 1);
+
+    QString newCode = code;
+    QList<QDeclarativeJS::AST::SourceLocation> pragmas = extractPragmas(newCode);
+    lexer.setCode(newCode, 1);
 
     return parser.parseProgram();
 }
@@ -111,7 +114,10 @@ QString JsCodeMarker::addMarkUp(const QString &code,
 {
     QDeclarativeJS::Engine engine;
     QDeclarativeJS::Lexer lexer(&engine);
-    lexer.setCode(code, 1);
+
+    QString newCode = code;
+    QList<QDeclarativeJS::AST::SourceLocation> pragmas = extractPragmas(newCode);
+    lexer.setCode(newCode, 1);
 
     QDeclarativeJS::Parser parser(&engine);
     QDeclarativeJS::NodePool m_nodePool("<JsCodeMarker::addMarkUp>", &engine);
@@ -119,7 +125,9 @@ QString JsCodeMarker::addMarkUp(const QString &code,
 
     if (parser.parseProgram()) {
         QDeclarativeJS::AST::Node *ast = parser.rootNode();
-        QmlMarkupVisitor visitor(code, &engine);
+        // Pass the unmodified code to the visitor so that pragmas and other
+        // unhandled source text can be output.
+        QmlMarkupVisitor visitor(code, pragmas, &engine);
         QDeclarativeJS::AST::Node::accept(ast, &visitor);
         output = visitor.markedUpCode();
     }
diff --git a/tools/qdoc3/qmlcodemarker.cpp b/tools/qdoc3/qmlcodemarker.cpp
new file mode 100644
index 0000000..1e4ad1e
--- /dev/null
+++ b/tools/qdoc3/qmlcodemarker.cpp
@@ -0,0 +1,287 @@
+/****************************************************************************
+**
+** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the tools applications of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights.  These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+/*
+  qmlcodemarker.cpp
+*/
+
+#include "private/qdeclarativejsast_p.h"
+#include "private/qdeclarativejsastfwd_p.h"
+#include "private/qdeclarativejsengine_p.h"
+#include "private/qdeclarativejslexer_p.h"
+#include "private/qdeclarativejsnodepool_p.h"
+#include "private/qdeclarativejsparser_p.h"
+
+#include "atom.h"
+#include "node.h"
+#include "qmlcodemarker.h"
+#include "qmlmarkupvisitor.h"
+#include "text.h"
+#include "tree.h"
+
+QT_BEGIN_NAMESPACE
+
+QmlCodeMarker::QmlCodeMarker()
+{
+}
+
+QmlCodeMarker::~QmlCodeMarker()
+{
+}
+
+/*!
+  Returns true if the \a code is recognized by the parser.
+ */
+bool QmlCodeMarker::recognizeCode(const QString &code)
+{
+    QDeclarativeJS::Engine engine;
+    QDeclarativeJS::Lexer lexer(&engine);
+    QDeclarativeJS::Parser parser(&engine);
+    QDeclarativeJS::NodePool m_nodePool("<QmlCodeMarker::recognizeCode>", &engine);
+
+    QString newCode = code;
+    extractPragmas(newCode);
+    lexer.setCode(newCode, 1);
+
+    return parser.parse();
+}
+
+/*!
+  Returns true if \a ext is any of a list of file extensions
+  for the QML language.
+ */
+bool QmlCodeMarker::recognizeExtension(const QString &ext)
+{
+    return ext == "qml";
+}
+
+/*!
+  Returns true if the \a language is recognized. Only "QML" is
+  recognized by this marker.
+ */
+bool QmlCodeMarker::recognizeLanguage(const QString &language)
+{
+    return language == "QML";
+}
+
+/*!
+  Returns the name of the \a node. Method names include are returned with a
+  trailing set of parentheses.
+ */
+QString QmlCodeMarker::plainName(const Node *node)
+{
+    QString name = node->name();
+    if (node->type() == Node::QmlMethod)
+        name += "()";
+    return name;
+}
+
+QString QmlCodeMarker::plainFullName(const Node *node, const Node *relative)
+{
+    if (node->name().isEmpty()) {
+        return "global";
+    }
+    else {
+        QString fullName;
+        while (node) {
+            fullName.prepend(plainName(node));
+            if (node->parent() == relative || node->parent()->name().isEmpty())
+                break;
+            fullName.prepend("::");
+            node = node->parent();
+        }
+        return fullName;
+    }
+}
+
+QString QmlCodeMarker::markedUpCode(const QString &code,
+                                    const Node *relative,
+                                    const QString &dirPath)
+{
+    return addMarkUp(code, relative, dirPath);
+}
+
+QString QmlCodeMarker::markedUpName(const Node *node)
+{
+    QString name = linkTag(node, taggedNode(node));
+    if (node->type() == Node::QmlMethod)
+        name += "()";
+    return name;
+}
+
+QString QmlCodeMarker::markedUpFullName(const Node *node, const Node *relative)
+{
+    if (node->name().isEmpty()) {
+        return "global";
+    }
+    else {
+        QString fullName;
+        for (;;) {
+            fullName.prepend(markedUpName(node));
+            if (node->parent() == relative || node->parent()->name().isEmpty())
+                break;
+            fullName.prepend("<@op>::</@op>");
+            node = node->parent();
+        }
+        return fullName;
+    }
+}
+
+QString QmlCodeMarker::markedUpIncludes(const QStringList& includes)
+{
+    QString code;
+
+    QStringList::ConstIterator inc = includes.begin();
+    while (inc != includes.end()) {
+        code += "import " + *inc + "\n";
+        ++inc;
+    }
+    return protect(addMarkUp(code, 0, ""));
+}
+
+QString QmlCodeMarker::functionBeginRegExp(const QString& funcName)
+{
+    return "^" + QRegExp::escape("function " + funcName) + "$";
+
+}
+
+QString QmlCodeMarker::functionEndRegExp(const QString& /* funcName */)
+{
+    return "^\\}$";
+}
+
+QString QmlCodeMarker::addMarkUp(const QString &code,
+                                 const Node * /* relative */,
+                                 const QString & /* dirPath */)
+{
+    QDeclarativeJS::Engine engine;
+    QDeclarativeJS::Lexer lexer(&engine);
+
+    QString newCode = code;
+    QList<QDeclarativeJS::AST::SourceLocation> pragmas = extractPragmas(newCode);
+    lexer.setCode(newCode, 1);
+
+    QDeclarativeJS::Parser parser(&engine);
+    QDeclarativeJS::NodePool m_nodePool("<QmlCodeMarker::addMarkUp>", &engine);
+    QString output;
+
+    if (parser.parse()) {
+        QDeclarativeJS::AST::UiProgram *ast = parser.ast();
+        // Pass the unmodified code to the visitor so that pragmas and other
+        // unhandled source text can be output.
+        QmlMarkupVisitor visitor(code, pragmas, &engine);
+        QDeclarativeJS::AST::Node::accept(ast, &visitor);
+        output = visitor.markedUpCode();
+    }
+    return output;
+}
+
+/*
+Copied and pasted from src/declarative/qml/qdeclarativescriptparser.cpp.
+*/
+static void replaceWithSpace(QString &str, int idx, int n)
+{
+    QChar *data = str.data() + idx;
+    const QChar space(QLatin1Char(' '));
+    for (int ii = 0; ii < n; ++ii)
+        *data++ = space;
+}
+
+/*
+Copied and pasted from src/declarative/qml/qdeclarativescriptparser.cpp then
+modified to return a list of removed pragmas.
+
+Searches for ".pragma <value>" declarations within \a script.  Currently supported pragmas
+are:
+    library
+*/
+QList<QDeclarativeJS::AST::SourceLocation> QmlCodeMarker::extractPragmas(QString &script)
+{
+    const QString pragma(QLatin1String("pragma"));
+    const QString library(QLatin1String("library"));
+    QList<QDeclarativeJS::AST::SourceLocation> removed;
+
+    QDeclarativeJS::Lexer l(0);
+    l.setCode(script, 0);
+
+    int token = l.lex();
+
+    while (true) {
+        if (token != QDeclarativeJSGrammar::T_DOT)
+            return removed;
+
+        int startOffset = l.tokenOffset();
+        int startLine = l.currentLineNo();
+        int startColumn = l.currentColumnNo();
+
+        token = l.lex();
+
+        if (token != QDeclarativeJSGrammar::T_IDENTIFIER ||
+            l.currentLineNo() != startLine ||
+            script.mid(l.tokenOffset(), l.tokenLength()) != pragma)
+            return removed;
+
+        token = l.lex();
+
+        if (token != QDeclarativeJSGrammar::T_IDENTIFIER ||
+            l.currentLineNo() != startLine)
+            return removed;
+
+        QString pragmaValue = script.mid(l.tokenOffset(), l.tokenLength());
+        int endOffset = l.tokenLength() + l.tokenOffset();
+
+        token = l.lex();
+        if (l.currentLineNo() == startLine)
+            return removed;
+
+        if (pragmaValue == QLatin1String("library")) {
+            replaceWithSpace(script, startOffset, endOffset - startOffset);
+            removed.append(
+                QDeclarativeJS::AST::SourceLocation(
+                    startOffset, endOffset - startOffset,
+                    startLine, startColumn));
+        } else
+            return removed;
+    }
+    return removed;
+}
+
+QT_END_NAMESPACE
diff --git a/tools/qdoc3/qmlcodemarker.h b/tools/qdoc3/qmlcodemarker.h
new file mode 100644
index 0000000..68e6753
--- /dev/null
+++ b/tools/qdoc3/qmlcodemarker.h
@@ -0,0 +1,85 @@
+/****************************************************************************
+**
+** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the tools applications of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights.  These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+/*
+  qmlcodemarker.h
+*/
+
+#ifndef QMLCODEMARKER_H
+#define QMLCODEMARKER_H
+
+#include "private/qdeclarativejsastfwd_p.h"
+#include "cppcodemarker.h"
+
+QT_BEGIN_NAMESPACE
+
+class QmlCodeMarker : public CppCodeMarker
+{
+public:
+    QmlCodeMarker();
+    ~QmlCodeMarker();
+
+    virtual bool recognizeCode(const QString &code);
+    virtual bool recognizeExtension(const QString &ext);
+    virtual bool recognizeLanguage(const QString &language);
+    virtual QString plainName(const Node *node);
+    virtual QString plainFullName(const Node *node, const Node *relative);
+    virtual QString markedUpCode(const QString &code, 
+                                 const Node *relative, 
+                                 const QString &dirPath);
+
+    virtual QString markedUpName(const Node *node);
+    virtual QString markedUpFullName(const Node *node, const Node *relative);
+    virtual QString markedUpIncludes(const QStringList &includes);
+    virtual QString functionBeginRegExp(const QString &funcName);
+    virtual QString functionEndRegExp(const QString &funcName);
+
+    /* Copied from src/declarative/qml/qdeclarativescriptparser.cpp */
+    QList<QDeclarativeJS::AST::SourceLocation> extractPragmas(QString &script);
+
+private:
+    QString addMarkUp(const QString &code, const Node * /* relative */,
+                      const QString & /* dirPath */);
+};
+
+QT_END_NAMESPACE
+
+#endif
diff --git a/tools/qdoc3/qmlcodeparser.cpp b/tools/qdoc3/qmlcodeparser.cpp
index 329912c..9c1d4ee 100644
--- a/tools/qdoc3/qmlcodeparser.cpp
+++ b/tools/qdoc3/qmlcodeparser.cpp
@@ -118,7 +118,10 @@ void QmlCodeParser::parseSourceFile(const Location& location,
     in.close();
 
     Location fileLocation(filePath);
-    lexer->setCode(document, 1);
+
+    QString newCode = document;
+    extractPragmas(newCode);
+    lexer->setCode(newCode, 1);
 
     QSet<QString> topicCommandsAllowed = topicCommands();
     QSet<QString> otherMetacommandsAllowed = otherMetaCommands();
@@ -129,7 +132,7 @@ void QmlCodeParser::parseSourceFile(const Location& location,
 
     if (parser->parse()) {
         QDeclarativeJS::AST::UiProgram *ast = parser->ast();
-        QmlDocVisitor visitor(filePath, document, &engine, tree, metacommandsAllowed);
+        QmlDocVisitor visitor(filePath, newCode, &engine, tree, metacommandsAllowed);
         QDeclarativeJS::AST::Node::accept(ast, &visitor);
     }
 }
@@ -165,4 +168,68 @@ QSet<QString> QmlCodeParser::otherMetaCommands()
                                 << COMMAND_QMLDEFAULT;
 }
 
+/*
+Copied and pasted from src/declarative/qml/qdeclarativescriptparser.cpp.
+*/
+static void replaceWithSpace(QString &str, int idx, int n) 
+{
+    QChar *data = str.data() + idx;
+    const QChar space(QLatin1Char(' '));
+    for (int ii = 0; ii < n; ++ii)
+        *data++ = space;
+}
+
+/*
+Copied and pasted from src/declarative/qml/qdeclarativescriptparser.cpp then
+modified to return no values.
+
+Searches for ".pragma <value>" declarations within \a script.  Currently supported pragmas
+are:
+    library
+*/
+void QmlCodeParser::extractPragmas(QString &script)
+{
+    const QString pragma(QLatin1String("pragma"));
+    const QString library(QLatin1String("library"));
+
+    QDeclarativeJS::Lexer l(0);
+    l.setCode(script, 0);
+
+    int token = l.lex();
+
+    while (true) {
+        if (token != QDeclarativeJSGrammar::T_DOT)
+            return;
+
+        int startOffset = l.tokenOffset();
+        int startLine = l.currentLineNo();
+
+        token = l.lex();
+
+        if (token != QDeclarativeJSGrammar::T_IDENTIFIER ||
+            l.currentLineNo() != startLine ||
+            script.mid(l.tokenOffset(), l.tokenLength()) != pragma)
+            return;
+
+        token = l.lex();
+
+        if (token != QDeclarativeJSGrammar::T_IDENTIFIER ||
+            l.currentLineNo() != startLine)
+            return;
+
+        QString pragmaValue = script.mid(l.tokenOffset(), l.tokenLength());
+        int endOffset = l.tokenLength() + l.tokenOffset();
+
+        token = l.lex();
+        if (l.currentLineNo() == startLine)
+            return;
+
+        if (pragmaValue == QLatin1String("library"))
+            replaceWithSpace(script, startOffset, endOffset - startOffset);
+        else
+            return;
+    }
+    return;
+}
+
 QT_END_NAMESPACE
diff --git a/tools/qdoc3/qmlcodeparser.h b/tools/qdoc3/qmlcodeparser.h
index a99e8a3..bbacd72 100644
--- a/tools/qdoc3/qmlcodeparser.h
+++ b/tools/qdoc3/qmlcodeparser.h
@@ -75,6 +75,9 @@ public:
                                  const QString& filePath, Tree *tree);
     virtual void doneParsingSourceFiles(Tree *tree);
 
+    /* Copied from src/declarative/qml/qdeclarativescriptparser.cpp */
+    void extractPragmas(QString &script);
+
 protected:
     virtual QSet<QString> topicCommands();
     virtual QSet<QString> otherMetaCommands();
diff --git a/tools/qdoc3/qmlmarkupvisitor.cpp b/tools/qdoc3/qmlmarkupvisitor.cpp
index 907b95d..6bede96 100644
--- a/tools/qdoc3/qmlmarkupvisitor.cpp
+++ b/tools/qdoc3/qmlmarkupvisitor.cpp
@@ -49,12 +49,42 @@
 
 QT_BEGIN_NAMESPACE
 
-QmlMarkupVisitor::QmlMarkupVisitor(const QString &source, QDeclarativeJS::Engine *engine)
+QmlMarkupVisitor::QmlMarkupVisitor(const QString &source,
+                  const QList<QDeclarativeJS::AST::SourceLocation> &pragmas,
+                  QDeclarativeJS::Engine *engine)
 {
     this->source = source;
     this->engine = engine;
+
     cursor = 0;
-    commentIndex = 0;
+    extraIndex = 0;
+
+    // Merge the lists of locations of pragmas and comments in the source code.
+    int i = 0;
+    int j = 0;
+    while (i < engine->comments().length() && j < pragmas.length()) {
+        if (engine->comments()[i].offset < pragmas[j].offset) {
+            extraTypes.append(Comment);
+            extraLocations.append(engine->comments()[i]);
+            ++i;
+        } else {
+            extraTypes.append(Pragma);
+            extraLocations.append(engine->comments()[j]);
+            ++j;
+        }
+    }
+
+    while (i < engine->comments().length()) {
+        extraTypes.append(Comment);
+        extraLocations.append(engine->comments()[i]);
+        ++i;
+    }
+
+    while (j < pragmas.length()) {
+        extraTypes.append(Pragma);
+        extraLocations.append(pragmas[j]);
+        ++j;
+    }
 }
 
 QmlMarkupVisitor::~QmlMarkupVisitor()
@@ -96,7 +126,7 @@ QString QmlMarkupVisitor::markedUpCode()
 
 void QmlMarkupVisitor::addExtra(quint32 start, quint32 finish)
 {
-    if (commentIndex >= engine->comments().length()) {
+    if (extraIndex >= extraLocations.length()) {
         QString extra = source.mid(start, finish - start);
         if (extra.trimmed().isEmpty())
             output += extra;
@@ -107,28 +137,37 @@ void QmlMarkupVisitor::addExtra(quint32 start, quint32 finish)
         return;
     }
 
-    while (commentIndex < engine->comments().length()) {
-        if (engine->comments()[commentIndex].offset - 2 >= start)
-            break;
-        commentIndex++;
+    while (extraIndex < extraLocations.length()) {
+        if (extraTypes[extraIndex] == Comment) {
+            if (extraLocations[extraIndex].offset - 2 >= start)
+                break;
+        } else {
+            if (extraLocations[extraIndex].offset >= start)
+                break;
+        }
+        extraIndex++;
     }
 
     quint32 i = start;
-    while (i < finish && commentIndex < engine->comments().length()) {
-        quint32 j = engine->comments()[commentIndex].offset - 2;
+    while (i < finish && extraIndex < extraLocations.length()) {
+        quint32 j = extraLocations[extraIndex].offset - 2;
         if (i <= j && j < finish) {
             if (i < j)
                 output += protect(source.mid(i, j - i));
 
-            quint32 l = engine->comments()[commentIndex].length;
-            if (source.mid(j, 2) == QLatin1String("/*"))
-                l += 4;
-            else
-                l += 2;
-            output += QLatin1String("<@comment>");
-            output += protect(source.mid(j, l));
-            output += QLatin1String("</@comment>");
-            commentIndex++;
+            quint32 l = extraLocations[extraIndex].length;
+            if (extraTypes[extraIndex] == Comment) {
+                if (source.mid(j, 2) == QLatin1String("/*"))
+                    l += 4;
+                else
+                    l += 2;
+                output += QLatin1String("<@comment>");
+                output += protect(source.mid(j, l));
+                output += QLatin1String("</@comment>");
+            } else
+                output += protect(source.mid(j, l));
+
+            extraIndex++;
             i = j + l;
         } else
             break;
diff --git a/tools/qdoc3/qmlmarkupvisitor.h b/tools/qdoc3/qmlmarkupvisitor.h
index 35f1b5bc..7a9ff22 100644
--- a/tools/qdoc3/qmlmarkupvisitor.h
+++ b/tools/qdoc3/qmlmarkupvisitor.h
@@ -52,7 +52,14 @@ QT_BEGIN_NAMESPACE
 class QmlMarkupVisitor : public QDeclarativeJS::AST::Visitor
 {
 public:
-    QmlMarkupVisitor(const QString &code, QDeclarativeJS::Engine *engine);
+    enum ExtraType{
+        Comment,
+        Pragma
+    };
+
+    QmlMarkupVisitor(const QString &code,
+                     const QList<QDeclarativeJS::AST::SourceLocation> &pragmas,
+                     QDeclarativeJS::Engine *engine);
     virtual ~QmlMarkupVisitor();
 
     QString markedUpCode();
@@ -160,10 +167,12 @@ private:
     QString sourceText(QDeclarativeJS::AST::SourceLocation &location);
 
     QDeclarativeJS::Engine *engine;
+    QList<ExtraType> extraTypes;
+    QList<QDeclarativeJS::AST::SourceLocation> extraLocations;
     QString source;
     QString output;
     quint32 cursor;
-    quint32 commentIndex;
+    int extraIndex;
 };
 
 QT_END_NAMESPACE