Long live Qt!

35 files changed, 370 insertions, 0 deletions

diff --git a/util/lexgen/tests/testdata/backtrack1/input b/util/lexgen/tests/testdata/backtrack1/input
new file mode 100644
index 0000000..f5099b5
--- /dev/null
+++ b/util/lexgen/tests/testdata/backtrack1/input
@@ -0,0 +1 @@
+LETX
diff --git a/util/lexgen/tests/testdata/backtrack1/output b/util/lexgen/tests/testdata/backtrack1/output
new file mode 100644
index 0000000..6893deb
--- /dev/null
+++ b/util/lexgen/tests/testdata/backtrack1/output
@@ -0,0 +1 @@
+TOK_LET|LET
diff --git a/util/lexgen/tests/testdata/backtrack1/rules.lexgen b/util/lexgen/tests/testdata/backtrack1/rules.lexgen
new file mode 100644
index 0000000..ade8a15
--- /dev/null
+++ b/util/lexgen/tests/testdata/backtrack1/rules.lexgen
@@ -0,0 +1,3 @@
+[Tokens]
+TOK_LET = LET
+TOK_LETXX = LETXX
diff --git a/util/lexgen/tests/testdata/backtrack2/input b/util/lexgen/tests/testdata/backtrack2/input
new file mode 100644
index 0000000..59ff5b7
--- /dev/null
+++ b/util/lexgen/tests/testdata/backtrack2/input
@@ -0,0 +1 @@
+LETXTRA
diff --git a/util/lexgen/tests/testdata/backtrack2/output b/util/lexgen/tests/testdata/backtrack2/output
new file mode 100644
index 0000000..348b382
--- /dev/null
+++ b/util/lexgen/tests/testdata/backtrack2/output
@@ -0,0 +1,2 @@
+TOK_LET|LET
+TOK_XTRA|XTRA
diff --git a/util/lexgen/tests/testdata/backtrack2/rules.lexgen b/util/lexgen/tests/testdata/backtrack2/rules.lexgen
new file mode 100644
index 0000000..6f16986
--- /dev/null
+++ b/util/lexgen/tests/testdata/backtrack2/rules.lexgen
@@ -0,0 +1,4 @@
+[Tokens]
+TOK_LET = LET
+TOK_LETXX = LETXX
+TOK_XTRA = XTRA
diff --git a/util/lexgen/tests/testdata/casesensitivity/input b/util/lexgen/tests/testdata/casesensitivity/input
new file mode 100644
index 0000000..72b7f48
--- /dev/null
+++ b/util/lexgen/tests/testdata/casesensitivity/input
@@ -0,0 +1 @@
+abcdAbcDABCDeFgEFGefgEfghiHIHihI
diff --git a/util/lexgen/tests/testdata/casesensitivity/output b/util/lexgen/tests/testdata/casesensitivity/output
new file mode 100644
index 0000000..3a4e819
--- /dev/null
+++ b/util/lexgen/tests/testdata/casesensitivity/output
@@ -0,0 +1,14 @@
+TOK_AB|ab
+TOK_CD|cd
+TOK_AB|Ab
+TOK_CD|cD
+TOK_AB|AB
+TOK_CD|CD
+TOK_EFG|eFg
+TOK_EFG|EFG
+TOK_EFG|efg
+TOK_EFG|Efg
+TOK_HI|hi
+TOK_HI|HI
+TOK_HI|Hi
+TOK_HI|hI
diff --git a/util/lexgen/tests/testdata/casesensitivity/rules.lexgen b/util/lexgen/tests/testdata/casesensitivity/rules.lexgen
new file mode 100644
index 0000000..3347587
--- /dev/null
+++ b/util/lexgen/tests/testdata/casesensitivity/rules.lexgen
@@ -0,0 +1,7 @@
+[Options]
+case-insensitive
+[Tokens]
+TOK_AB = ab
+TOK_CD = cd
+TOK_EFG = [e-g]{3}
+TOK_HI = [hi]{2}
diff --git a/util/lexgen/tests/testdata/comments/input b/util/lexgen/tests/testdata/comments/input
new file mode 100644
index 0000000..03873e0
--- /dev/null
+++ b/util/lexgen/tests/testdata/comments/input
@@ -0,0 +1 @@
+/* comment with stuff *//*another comment with * stars * inside*/
diff --git a/util/lexgen/tests/testdata/comments/output b/util/lexgen/tests/testdata/comments/output
new file mode 100644
index 0000000..2395ad1
--- /dev/null
+++ b/util/lexgen/tests/testdata/comments/output
@@ -0,0 +1,2 @@
+TOK_COMMENT|/* comment with stuff */
+TOK_COMMENT|/*another comment with * stars * inside*/
diff --git a/util/lexgen/tests/testdata/comments/rules.lexgen b/util/lexgen/tests/testdata/comments/rules.lexgen
new file mode 100644
index 0000000..490c759
--- /dev/null
+++ b/util/lexgen/tests/testdata/comments/rules.lexgen
@@ -0,0 +1,2 @@
+[Tokens]
+TOK_COMMENT = \/\*[^*]*\*+([^/*][^*]*\*+)*\/
diff --git a/util/lexgen/tests/testdata/dot/input b/util/lexgen/tests/testdata/dot/input
new file mode 100644
index 0000000..e5b0ad6
--- /dev/null
+++ b/util/lexgen/tests/testdata/dot/input
@@ -0,0 +1 @@
+afbcxd
diff --git a/util/lexgen/tests/testdata/dot/output b/util/lexgen/tests/testdata/dot/output
new file mode 100644
index 0000000..6a9afd4
--- /dev/null
+++ b/util/lexgen/tests/testdata/dot/output
@@ -0,0 +1,2 @@
+TOK_AB|afb
+TOK_CD|cxd
diff --git a/util/lexgen/tests/testdata/dot/rules.lexgen b/util/lexgen/tests/testdata/dot/rules.lexgen
new file mode 100644
index 0000000..03873a7
--- /dev/null
+++ b/util/lexgen/tests/testdata/dot/rules.lexgen
@@ -0,0 +1,3 @@
+[Tokens]
+TOK_AB = a.b
+TOK_CD = c.d
diff --git a/util/lexgen/tests/testdata/negation/input b/util/lexgen/tests/testdata/negation/input
new file mode 100644
index 0000000..9447b80
--- /dev/null
+++ b/util/lexgen/tests/testdata/negation/input
@@ -0,0 +1 @@
+aycabd
diff --git a/util/lexgen/tests/testdata/negation/output b/util/lexgen/tests/testdata/negation/output
new file mode 100644
index 0000000..0b73263
--- /dev/null
+++ b/util/lexgen/tests/testdata/negation/output
@@ -0,0 +1,2 @@
+TOK_A|ayc
+TOK_B|abd
diff --git a/util/lexgen/tests/testdata/negation/rules.lexgen b/util/lexgen/tests/testdata/negation/rules.lexgen
new file mode 100644
index 0000000..179810b
--- /dev/null
+++ b/util/lexgen/tests/testdata/negation/rules.lexgen
@@ -0,0 +1,3 @@
+[Tokens]
+TOK_A = a[^b]c
+TOK_B = abd
diff --git a/util/lexgen/tests/testdata/quoteinset/input b/util/lexgen/tests/testdata/quoteinset/input
new file mode 100644
index 0000000..5a9b680
--- /dev/null
+++ b/util/lexgen/tests/testdata/quoteinset/input
@@ -0,0 +1 @@
+"a
diff --git a/util/lexgen/tests/testdata/quoteinset/output b/util/lexgen/tests/testdata/quoteinset/output
new file mode 100644
index 0000000..7ba8890
--- /dev/null
+++ b/util/lexgen/tests/testdata/quoteinset/output
@@ -0,0 +1 @@
+TOK_QUOTEA|"a
diff --git a/util/lexgen/tests/testdata/quoteinset/rules.lexgen b/util/lexgen/tests/testdata/quoteinset/rules.lexgen
new file mode 100644
index 0000000..9838276
--- /dev/null
+++ b/util/lexgen/tests/testdata/quoteinset/rules.lexgen
@@ -0,0 +1,2 @@
+[Tokens]
+TOK_QUOTEA = ["]a
diff --git a/util/lexgen/tests/testdata/quotes/input b/util/lexgen/tests/testdata/quotes/input
new file mode 100644
index 0000000..ac54450
--- /dev/null
+++ b/util/lexgen/tests/testdata/quotes/input
@@ -0,0 +1 @@
+quotedstring
diff --git a/util/lexgen/tests/testdata/quotes/output b/util/lexgen/tests/testdata/quotes/output
new file mode 100644
index 0000000..c538e32
--- /dev/null
+++ b/util/lexgen/tests/testdata/quotes/output
@@ -0,0 +1 @@
+TOK_STR|quotedstring
diff --git a/util/lexgen/tests/testdata/quotes/rules.lexgen b/util/lexgen/tests/testdata/quotes/rules.lexgen
new file mode 100644
index 0000000..d528cdd
--- /dev/null
+++ b/util/lexgen/tests/testdata/quotes/rules.lexgen
@@ -0,0 +1,2 @@
+[Tokens]
+TOK_STR = "quotedstring"
diff --git a/util/lexgen/tests/testdata/simple/input b/util/lexgen/tests/testdata/simple/input
new file mode 100644
index 0000000..acbe86c
--- /dev/null
+++ b/util/lexgen/tests/testdata/simple/input
@@ -0,0 +1 @@
+abcd
diff --git a/util/lexgen/tests/testdata/simple/output b/util/lexgen/tests/testdata/simple/output
new file mode 100644
index 0000000..a37a58e
--- /dev/null
+++ b/util/lexgen/tests/testdata/simple/output
@@ -0,0 +1,2 @@
+TOK_AB|ab
+TOK_CD|cd
diff --git a/util/lexgen/tests/testdata/simple/rules.lexgen b/util/lexgen/tests/testdata/simple/rules.lexgen
new file mode 100644
index 0000000..5d958c4
--- /dev/null
+++ b/util/lexgen/tests/testdata/simple/rules.lexgen
@@ -0,0 +1,3 @@
+[Tokens]
+TOK_AB = ab
+TOK_CD = cd
diff --git a/util/lexgen/tests/testdata/subsets1/input b/util/lexgen/tests/testdata/subsets1/input
new file mode 100644
index 0000000..47f66d1
--- /dev/null
+++ b/util/lexgen/tests/testdata/subsets1/input
@@ -0,0 +1 @@
+abaf
diff --git a/util/lexgen/tests/testdata/subsets1/output b/util/lexgen/tests/testdata/subsets1/output
new file mode 100644
index 0000000..75dd936
--- /dev/null
+++ b/util/lexgen/tests/testdata/subsets1/output
@@ -0,0 +1,2 @@
+TOK_AB|ab
+TOK_AZ|af
diff --git a/util/lexgen/tests/testdata/subsets1/rules.lexgen b/util/lexgen/tests/testdata/subsets1/rules.lexgen
new file mode 100644
index 0000000..94f51a9
--- /dev/null
+++ b/util/lexgen/tests/testdata/subsets1/rules.lexgen
@@ -0,0 +1,3 @@
+[Tokens]
+TOK_AB = ab
+TOK_AZ = a[a-z]
diff --git a/util/lexgen/tests/testdata/subsets2/input b/util/lexgen/tests/testdata/subsets2/input
new file mode 100644
index 0000000..4d0dc3a
--- /dev/null
+++ b/util/lexgen/tests/testdata/subsets2/input
@@ -0,0 +1 @@
+abd
diff --git a/util/lexgen/tests/testdata/subsets2/output b/util/lexgen/tests/testdata/subsets2/output
new file mode 100644
index 0000000..d5a7bc5
--- /dev/null
+++ b/util/lexgen/tests/testdata/subsets2/output
@@ -0,0 +1,3 @@
+TOK_A|a
+TOK_B|b
+TOK_D|d
diff --git a/util/lexgen/tests/testdata/subsets2/rules.lexgen b/util/lexgen/tests/testdata/subsets2/rules.lexgen
new file mode 100644
index 0000000..e0a7629
--- /dev/null
+++ b/util/lexgen/tests/testdata/subsets2/rules.lexgen
@@ -0,0 +1,4 @@
+[Tokens]
+TOK_D = [abcd]
+TOK_B = [bc]
+TOK_A = a
diff --git a/util/lexgen/tests/tests.pro b/util/lexgen/tests/tests.pro
new file mode 100644
index 0000000..eb04439
--- /dev/null
+++ b/util/lexgen/tests/tests.pro
@@ -0,0 +1,6 @@
+CONFIG += qtestlib
+SOURCES += tst_lexgen.cpp
+TARGET = tst_lexgen
+include(../lexgen.pri)
+QT = core
+DEFINES += SRCDIR=\\\"$$PWD\\\"
diff --git a/util/lexgen/tests/tst_lexgen.cpp b/util/lexgen/tests/tst_lexgen.cpp
new file mode 100644
index 0000000..6e50b15
--- /dev/null
+++ b/util/lexgen/tests/tst_lexgen.cpp
@@ -0,0 +1,285 @@
+/****************************************************************************
+**
+** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: Qt Software Information (qt-info@nokia.com)
+**
+** This file is part of the utils of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the either Technology Preview License Agreement or the
+** Beta Release License Agreement.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain
+** additional rights. These rights are described in the Nokia Qt LGPL
+** Exception version 1.0, included in the file LGPL_EXCEPTION.txt in this
+** package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3.0 as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU General Public License version 3.0 requirements will be
+** met: http://www.gnu.org/copyleft/gpl.html.
+**
+** If you are unsure which license is appropriate for your use, please
+** contact the sales department at qt-sales@nokia.com.
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <QtTest/QtTest>
+#define AUTOTEST
+#include "../main.cpp"
+
+class tst_LexGen : public QObject
+{
+    Q_OBJECT
+private slots:
+    void nfa_singleInput();
+    void nfa_alternating();
+    void nfa_concatenating();
+    void nfa_optional();
+    void nfa_toDFA_data();
+    void nfa_toDFA();
+    void lexgen_data();
+    void lexgen();
+};
+
+void tst_LexGen::nfa_singleInput()
+{
+    NFA nfa = NFA::createSingleInputNFA('a');
+
+    QCOMPARE(nfa.initialState, 0);
+    QCOMPARE(nfa.finalState, 1);
+
+    QCOMPARE(nfa.states.count(), 2);
+
+    QCOMPARE(nfa.states.at(0).transitions.count(), 1);
+    QVERIFY(nfa.states.at(0).transitions.contains('a'));
+    QCOMPARE(nfa.states.at(0).transitions.values('a').count(), 1);
+    QCOMPARE(nfa.states.at(0).transitions.value('a'), nfa.finalState);
+
+    QVERIFY(nfa.states.at(1).transitions.isEmpty());
+}
+
+void tst_LexGen::nfa_alternating()
+{
+    NFA a = NFA::createSingleInputNFA('a');
+    NFA b = NFA::createSingleInputNFA('b');
+    NFA nfa = NFA::createAlternatingNFA(a, b);
+
+    const int initialA = 1;
+    const int finalA = 2;
+
+    const int initialB = 3;
+    const int finalB = 4;
+
+    QCOMPARE(nfa.states.count(), 6);
+
+    QCOMPARE(nfa.initialState, 0);
+    QCOMPARE(nfa.finalState, 5);
+
+    QList<int> initialTransitions = nfa.states.at(0).transitions.values(Epsilon);
+    QCOMPARE(initialTransitions.count(), 2);
+    QVERIFY(initialTransitions.contains(initialA));
+    QVERIFY(initialTransitions.contains(initialB));
+
+    // no need to test the individual a and b NFAs, the other
+    // autotest already takes care of that. Just check whether
+    // the epsilon transitions to the final state exist.
+
+    QCOMPARE(nfa.states.at(finalA).transitions.count(), 1);
+    QCOMPARE(nfa.states.at(finalA).transitions.values(Epsilon).count(), 1);
+    QCOMPARE(nfa.states.at(finalA).transitions.value(Epsilon), nfa.finalState);
+
+    QCOMPARE(nfa.states.at(finalB).transitions.count(), 1);
+    QCOMPARE(nfa.states.at(finalB).transitions.values(Epsilon).count(), 1);
+    QCOMPARE(nfa.states.at(finalB).transitions.value(Epsilon), nfa.finalState);
+}
+
+void tst_LexGen::nfa_concatenating()
+{
+    NFA a = NFA::createSingleInputNFA('a');
+    NFA b = NFA::createSingleInputNFA('b');
+    NFA nfa = NFA::createConcatenatingNFA(a, b);
+
+    const int initialA = 1;
+    const int finalA = 2;
+
+    const int initialB = 3;
+    const int finalB = 4;
+
+    QCOMPARE(nfa.states.count(), 6);
+
+    QCOMPARE(nfa.initialState, 0);
+    QCOMPARE(nfa.finalState, 5);
+
+    QCOMPARE(nfa.states.at(0).transitions.count(), 1);
+    QCOMPARE(nfa.states.at(0).transitions.values(Epsilon).count(), 1);
+    QCOMPARE(nfa.states.at(0).transitions.value(Epsilon), initialA);
+
+    QCOMPARE(nfa.states.at(finalA).transitions.values(Epsilon).count(), 1);
+    QCOMPARE(nfa.states.at(finalA).transitions.value(Epsilon), initialB);
+
+    QCOMPARE(nfa.states.at(finalB).transitions.values(Epsilon).count(), 1);
+    QCOMPARE(nfa.states.at(finalB).transitions.value(Epsilon), nfa.finalState);
+}
+
+void tst_LexGen::nfa_optional()
+{
+    NFA a = NFA::createSingleInputNFA('a');
+    NFA nfa = NFA::createOptionalNFA(a);
+
+    const int initialA = 1;
+    const int finalA = 2;
+
+    QCOMPARE(nfa.states.count(), 4);
+
+    QCOMPARE(nfa.initialState, 0);
+    QCOMPARE(nfa.finalState, 3);
+
+    QCOMPARE(nfa.states.at(0).transitions.count(), 2);
+    QList<int> initialTransitions = nfa.states.at(0).transitions.values(Epsilon);
+    QVERIFY(initialTransitions.contains(nfa.finalState));
+    QVERIFY(initialTransitions.contains(initialA));
+
+    QList<int> finalEpsilonATransitions = nfa.states.at(finalA).transitions.values(Epsilon);
+    QVERIFY(finalEpsilonATransitions.contains(initialA));
+    QVERIFY(finalEpsilonATransitions.contains(nfa.finalState));
+}
+
+Q_DECLARE_METATYPE(NFA);
+Q_DECLARE_METATYPE(DFA);
+
+void tst_LexGen::nfa_toDFA_data()
+{
+    QTest::addColumn<NFA>("nfa");
+    QTest::addColumn<DFA>("expectedDFA");
+
+    NFA a = NFA::createSingleInputNFA('a');
+    NFA b = NFA::createSingleInputNFA('b');
+    NFA c = NFA::createSingleInputNFA('c');
+
+    NFA nfa;
+    DFA dfa;
+
+    dfa.clear();
+    dfa.resize(3);
+    dfa[0].transitions.insert('a', 1);
+    dfa[1].transitions.insert('b', 2);
+
+    nfa = NFA::createConcatenatingNFA(a, b);
+
+    QTest::newRow("simple concat") << nfa << dfa;
+
+    dfa.clear();
+    dfa.resize(3);
+    dfa[0].transitions.insert('a', 1);
+    dfa[0].transitions.insert('b', 2);
+
+    nfa = NFA::createAlternatingNFA(a, b);
+
+    QTest::newRow("simple alternate") << nfa << dfa;
+
+}
+
+void tst_LexGen::nfa_toDFA()
+{
+    QFETCH(NFA, nfa);
+    QFETCH(DFA, expectedDFA);
+
+    DFA dfa = nfa.toDFA();
+
+    QCOMPARE(dfa.count(), expectedDFA.count());
+    for (int i = 0; i < dfa.count(); ++i) {
+        if (dfa.at(i).transitions != expectedDFA.at(i).transitions) {
+            qDebug() << "DFAs differ in state" << i;
+            qDebug() << "NFA:";
+            nfa.debug();
+            qDebug() << "Actual DFA:";
+            dfa.debug();
+            qDebug() << "Expected DFA:";
+            expectedDFA.debug();
+            QVERIFY(false);
+        }
+    }
+}
+
+void tst_LexGen::lexgen_data()
+{
+    QTest::addColumn<QString>("ruleFile");
+    QTest::addColumn<QString>("input");
+    QTest::addColumn<QString>("expectedOutput");
+
+    QDir d(QString(SRCDIR));
+    d.cd("testdata");
+    foreach (QString test, d.entryList(QDir::Dirs | QDir::NoDotAndDotDot)) {
+        QString dir = d.absoluteFilePath(test) + '/';
+        QTest::newRow(qPrintable(test)) 
+            << dir + "rules.lexgen"
+            << dir + "input"
+            << dir + "output"
+            ;
+    }
+}
+
+void tst_LexGen::lexgen()
+{
+    QFETCH(QString, ruleFile);
+    QFETCH(QString, input);
+    QFETCH(QString, expectedOutput);
+
+    Config conf;
+    QVERIFY(loadConfig(ruleFile, &conf));
+    DFA dfa = generateMachine(conf);
+    QVERIFY(!dfa.isEmpty());
+    conf.debug = true;
+
+    QFile f(input);
+    QVERIFY(f.open(QIODevice::ReadOnly));
+    input = QString::fromUtf8(f.readAll());
+    f.close();
+    if (input.endsWith(QLatin1Char('\n')))
+        input.chop(1);
+//    machine.debug();
+    bool ok = false;
+    QList<Symbol> symbols = tokenize(dfa, input, &conf, &ok);
+    QVERIFY(ok);
+    f.setFileName(expectedOutput);
+    QVERIFY(f.open(QIODevice::ReadOnly));
+    QStringList lines;
+    while (!f.atEnd()) {
+        QString line = QString::fromUtf8(f.readLine());
+        if (line.endsWith(QLatin1Char('\n')))
+            line.chop(1);
+        lines << line;
+    }
+    f.close();
+
+//    dfa.debug();
+    QCOMPARE(lines.count(), symbols.count());
+
+    for (int i = 0; i < lines.count(); ++i) {
+        QStringList l = lines.at(i).split(QChar::fromLatin1('|'));
+        QCOMPARE(l.count(), 2);
+        QString expectedToken = l.at(0);
+        QString expectedLexem = l.at(1);
+        QCOMPARE(symbols.at(i).token, expectedToken);
+        QCOMPARE(symbols.at(i).lexem, expectedLexem);
+    }
+}
+
+QTEST_MAIN(tst_LexGen)
+#include "tst_lexgen.moc"