BEnchmarks and performance improvements

17 files changed, 1108 insertions, 56 deletions

diff --git a/contrib/benchmarks/LCCA.pdf b/contrib/benchmarks/LCCA.pdf
new file mode 100644
index 0000000..4d5c051
--- /dev/null
+++ b/contrib/benchmarks/LCCA.pdf
diff --git a/contrib/benchmarks/Transitions.pdf b/contrib/benchmarks/Transitions.pdf
new file mode 100644
index 0000000..34b0ef7
--- /dev/null
+++ b/contrib/benchmarks/Transitions.pdf
diff --git a/contrib/benchmarks/apache-commons/benchmark.sh b/contrib/benchmarks/apache-commons/benchmark.sh
deleted file mode 100755
index ec312a1..0000000
--- a/contrib/benchmarks/apache-commons/benchmark.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/sh
-
-set -e
-
-# mvn archetype:generate -DgroupId=org.uscxml.benchmark -DartifactId=benchmark -DarchetypeArtifactId=maven-archetype-quickstart -DinteractiveMode=false
-
-mvn test
\ No newline at end of file
diff --git a/contrib/benchmarks/apache-commons/pom.xml b/contrib/benchmarks/apache/pom.xml
index 58f6f14..58f6f14 100644
--- a/contrib/benchmarks/apache-commons/pom.xml
+++ b/contrib/benchmarks/apache/pom.xml
diff --git a/contrib/benchmarks/apache-commons/src/test/java/org/uscxml/benchmark/BenchmarkTest.java b/contrib/benchmarks/apache/src/test/java/org/uscxml/benchmark/BenchmarkTest.java
index 0888646..08a9a38 100644
--- a/contrib/benchmarks/apache-commons/src/test/java/org/uscxml/benchmark/BenchmarkTest.java
+++ b/contrib/benchmarks/apache/src/test/java/org/uscxml/benchmark/BenchmarkTest.java
@@ -27,16 +27,18 @@ import org.apache.commons.scxml2.model.TransitionTarget;
  */
 public class BenchmarkTest extends TestCase {
 
+    public long initMs = 0;
+
     class PerformanceListener extends SimpleErrorReporter implements SCXMLListener {
         public long iterations = 0;
         public long mark = System.currentTimeMillis();
 
         public void onEntry(final EnterableState state) {
-            if (state.getId().equals("p0")) {
+            if (state.getId().equals("mark")) {
                 iterations++;
                 long now = System.currentTimeMillis();
                 if (now - mark > 1000) {
-                    System.out.println(iterations);
+                    System.out.println(initMs + ", " + iterations);
                     mark = now;
                     iterations = 0;
                 }
@@ -85,8 +87,11 @@ public class BenchmarkTest extends TestCase {
     public void testApp()
     {
         try {
-            SCXML scxml = parse(new URL("file:../../../test/benchmarks/findLCCA.scxml"), null);
+            long started = System.currentTimeMillis();
+            String fileName = System.getenv("USCXML_BENCHMARK");
+            SCXML scxml = parse(new URL("file:" + fileName), null);
             SCXMLExecutor exec = getExecutor(scxml, null, new SimpleDispatcher());
+            initMs = System.currentTimeMillis() - started;
             exec.go();
         } catch (Exception e) {
             System.err.println(e);
diff --git a/contrib/benchmarks/gnuplot-style.plt b/contrib/benchmarks/gnuplot-style.plt
new file mode 100644
index 0000000..05cc0b1
--- /dev/null
+++ b/contrib/benchmarks/gnuplot-style.plt
@@ -0,0 +1,36 @@
+set loadpath \
+	'~/Documents/TK/Code/gnuplot-colorbrewer/diverging' \
+	'~/Documents/TK/Code/gnuplot-colorbrewer/qualitative' \
+	'~/Documents/TK/Code/gnuplot-colorbrewer/sequential'
+
+load 'Dark2.plt'
+
+set datafile missing "NA"
+
+# set style line 1 lc rgb '#1B9E77' lw 1.5 pt 5 ps 0.2
+# set style line 2 lc rgb '#1B9E77' lw 1.5 pt 5 ps 0.2 dt 4
+# set style line 3 lc rgb '#D95F02' lw 1.5 pt 5 ps 0.2
+# set style line 4 lc rgb '#D95F02' lw 1.5 pt 5 ps 0.2 dt 4
+# set style line 5 lc rgb '#7570B3' lw 1.5 pt 5 ps 0.2
+# set style line 6 lc rgb '#7570B3' lw 1.5 pt 5 ps 0.2 dt 4
+# set style line 7 lc rgb '#E7298A' lw 1.5 pt 5 ps 0.2
+# set style line 8 lc rgb '#E7298A' lw 1.5 pt 5 ps 0.2 dt 4
+# set style line 20 lc rgb '#666666' lw 1.5 pt 5 ps 0.2 dt 4
+
+set terminal pdf font 'Verdana,10'
+# set termoption dash
+# set palette negative
+set termoption enhanced
+
+# deemphasize border
+set style line 11 lc rgb '#505050' lt 4
+set border 11 back ls 11
+
+# small grid
+set style line 12 lc rgb '#808080' lt 0 lw 1
+set grid ytics back ls 12
+
+set xtics nomirror out
+# set ytics nomirror
+
+set xtics border rotate by -45
diff --git a/contrib/benchmarks/lxsc/benchmark.sh b/contrib/benchmarks/lxsc/benchmark.sh
deleted file mode 100755
index d0860ad..0000000
--- a/contrib/benchmarks/lxsc/benchmark.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-set -e
-
-git clone https://github.com/Phrogz/LXSC.git
-cp ../../../test/benchmarks/findLCCA.scxml ./test.scxml
-
-lua ./test-performance.lua
\ No newline at end of file
diff --git a/contrib/benchmarks/lxsc/statesPerSecond.lua b/contrib/benchmarks/lxsc/statesPerSecond.lua
new file mode 100644
index 0000000..8469ea5
--- /dev/null
+++ b/contrib/benchmarks/lxsc/statesPerSecond.lua
@@ -0,0 +1,32 @@
+#!/usr/bin/env lua
+
+package.path = 'LXSC/?.lua;' .. package.path
+
+require 'io'
+require 'os'
+local LXSC = require 'lxsc'
+local file = ...
+local init, mark, now
+local iterations = 0
+
+local started = os.clock()
+local xml = io.open(file):read("*all")
+lxsc = LXSC:parse(xml)
+
+lxsc.onAfterEnter = function(id,kind) 
+	if (id=="mark") then
+		iterations = iterations + 1
+		now = os.clock()
+		if (now - mark > 1) then
+			print(init .. ", " .. iterations)
+			mark = now
+			iterations = 0
+		end
+	end
+end
+
+now = os.clock()
+init = (now - started) * 1000;
+mark = now
+
+lxsc:start()
diff --git a/contrib/benchmarks/lxsc/test-performance.lua b/contrib/benchmarks/lxsc/test-performance.lua
deleted file mode 100644
index 111fffa..0000000
--- a/contrib/benchmarks/lxsc/test-performance.lua
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env lua
-
-package.path = 'LXSC/?.lua;' .. package.path
-
-require 'io'
-require 'os'
-local LXSC = require 'lxsc'
-
-local c,t,lxsc = os.clock
-
-local out = io.open(string.format("results-%s.txt",LXSC.VERSION),"w")
-local sum=0
-function mark(msg,t2,n)
-  local delta = (t2-t)*1000/(n or 1)
-  sum = sum + delta
-  out:write(string.format("%25s: %5.2fms\n",msg,delta))  
-end
-
-local xml = io.open("test.scxml"):read("*all")
-t = c()
-for i=1,20 do lxsc = LXSC:parse(xml) end
-mark("Parse XML",c(),20)
-
-lxsc.onAfterEnter = function(id,kind) 
-	if (id=="id401") then
-		print("Entered "..kind.." '"..tostring(id).."'") 
-	end
-end
-
-t = c()
-lxsc:start()
-mark("Start Machine",c())
-
-
-out:write("----------------------------------\n")
-out:write(string.format("%25s: %5.2fms ± 20%%\n","Total time",sum))  
-
-out:close()
\ No newline at end of file
diff --git a/contrib/benchmarks/plot.sh b/contrib/benchmarks/plot.sh
new file mode 100755
index 0000000..c91ef70
--- /dev/null
+++ b/contrib/benchmarks/plot.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+for bench in Transitions LCCA; do
+	rm logs/${bench}.log
+
+	for size in 4 16 64 256 512; do
+
+		for impl in lxsc scion qt scxmlcc apache uscxml-fast uscxml-large; do
+			log="logs/${bench}.${size}-${impl}.log"
+			if [ ! -s ${log} ]; then
+				for i in {1..10}; do
+					echo 0, 0 >> $log
+				done
+			fi
+
+			# prepend implementation name as header
+			echo "${impl}.init.${size}", "${impl}" > ${log}.header
+			cat ${log} >> ${log}.header
+
+			# analyze values with R
+R -q -e "\
+library(psych); \
+x <- read.csv('${log}.header', head=TRUE, sep=","); \
+y <- describe(x, trim=.1, skew=FALSE); \
+write.table(y, file = \"${log}.r.csv\", sep=', ');"
+			tail -n2 ${log}.r.csv >> logs/${bench}.log
+			rm ${log}.header
+		done
+	done
+
+awk 'NR == 0 || NR % 2 == 0' logs/${bench}.log > logs/${bench}.even
+awk 'NR == 0 || NR % 2 == 1' logs/${bench}.log > logs/${bench}.odd
+paste -d', ' logs/${bench}.even /dev/null logs/${bench}.odd > logs/${bench}.joined.log
+
+cat << END_GNUPLOT > /tmp/tmp.plot
+	set title "Iterations per second for ${bench} Benchmark"
+
+	load 'gnuplot-style.plt'
+	set boxwidth 0.25 relative
+	set style fill solid 0.25 border
+	set key right top
+	set offsets 0.5,0.5,0,0
+
+	# set yrange [0:*]
+	# set y2range [0:5000]
+	# set y2tics nomirror
+	set ytics nomirror
+	set xtics ("4^2" 0, "16^2" 1, "64^2" 2, "256^2" 3, "512^2" 4)
+
+	set logscale y
+
+	set ylabel "Iterations / sec"
+	set xlabel "Complexity"
+	# set y2label "Initialization [ms]"
+
+	# see also https://stackoverflow.com/a/25512858/990120
+	plot 'logs/${bench}.joined.log' \
+		   using (\$0):4:5 every 7::6 with yerrorlines title "uscxml large" axis x1y1, \
+		'' using (\$0):4:5 every 7::5 with yerrorlines title "uscxml fast" axis x1y1, \
+		'' using (\$0):4:5 every 7::3 with yerrorlines title "scxmlcc" axis x1y1, \
+		'' using (\$0):4:5 every 7::4 with yerrorlines title "apache" axis x1y1, \
+		'' using (\$0):4:5 every 7::1 with yerrorlines title "scion" axis x1y1, \
+		'' using (\$0):4:5 every 7::0 with yerrorlines title "lxsc" axis x1y1, \
+		'' using (\$0):4:5 every 7::2 with yerrorlines title "qt" axis x1y1, \
+		# '' using (\$0 + 0.1):4:5:xtic(1) every 2::1 with yerrorlines title "Iterations per Second" axis x1y1 #, \
+		#'' using (\$0 + 0.1):(\$4):(sprintf("%d", \$4)) every 2::1 with labels notitle offset char 0,1
+END_GNUPLOT
+
+	gnuplot /tmp/tmp.plot > "logs/${bench}.pdf"
+
+
+done;
diff --git a/contrib/benchmarks/qt/StatesPerSecond.cpp b/contrib/benchmarks/qt/StatesPerSecond.cpp
new file mode 100644
index 0000000..d79a245
--- /dev/null
+++ b/contrib/benchmarks/qt/StatesPerSecond.cpp
@@ -0,0 +1,401 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtScxml module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <QtTest>
+#include <QObject>
+#include <QXmlStreamReader>
+#include <QtScxml/qscxmlcompiler.h>
+#include <QtScxml/qscxmlstatemachine.h>
+
+#include <iostream>
+#include <cstdlib>
+
+Q_DECLARE_METATYPE(QScxmlError);
+
+enum { SpyWaitTime = 8000 };
+
+class StatesPerSecond: public QObject
+{
+    Q_OBJECT
+
+private Q_SLOTS:
+    void connections();
+#if 0
+    void stateNames_data();
+    void stateNames();
+    void activeStateNames_data();
+    void activeStateNames();
+    void onExit();
+    void eventOccurred();
+
+    void doneDotStateEvent();
+    void running();
+#endif
+};
+
+void StatesPerSecond::connections()
+{
+    QElapsedTimer timer;
+    QElapsedTimer duration;
+    timer.start();
+    QScopedPointer<QScxmlStateMachine> stateMachine(
+                QScxmlStateMachine::fromFile(QString(getenv("USCXML_BENCHMARK"))));
+    std::cout << getenv("USCXML_BENCHMARK") << std::endl;
+    QVERIFY(!stateMachine.isNull());
+    size_t initMs = 0;
+    size_t iterations = 0;
+    QMetaObject::Connection final = stateMachine->connectToState("mark",
+                                                                 [&iterations, &timer, &duration, &initMs](bool enabled) {
+        if (!enabled)
+          return;
+        iterations++;
+        if (timer.elapsed() > 1000) {
+            std::cout << initMs << ", " << iterations << std::endl;
+            iterations = 0;
+            timer.restart();
+        }
+        if (duration.elapsed() > 10000)
+            exit(0);
+    });
+    QVERIFY(final);
+
+    initMs = timer.elapsed();
+
+    timer.restart();
+    duration.start();
+    stateMachine->start();
+
+    QTRY_VERIFY(iterations > 0);
+
+#if 0
+    Receiver receiver;
+
+    bool a1Reached = false;
+    bool finalReached = false;
+    QMetaObject::Connection a = stateMachine->connectToState("a", &receiver, &Receiver::a);
+    QVERIFY(a);
+    QMetaObject::Connection b = stateMachine->connectToState("b", &receiver, SLOT(b(bool)));
+    QVERIFY(b);
+    QMetaObject::Connection a1 = stateMachine->connectToState("a1", &receiver,
+                                                              [&a1Reached](bool enabled) {
+        a1Reached = a1Reached || enabled;
+    });
+    QVERIFY(a1);
+    QMetaObject::Connection final = stateMachine->connectToState("final",
+                                                                 [&finalReached](bool enabled) {
+        finalReached = finalReached || enabled;
+    });
+    QVERIFY(final);
+
+    bool a1Entered = false;
+    bool a1Exited = false;
+    bool finalEntered = false;
+    bool finalExited = false;
+    typedef QScxmlStateMachine QXSM;
+
+    QMetaObject::Connection aEntry = stateMachine->connectToState(
+                "a", QXSM::onEntry(&receiver, &Receiver::aEnter));
+    QVERIFY(aEntry);
+    QMetaObject::Connection aExit = stateMachine->connectToState(
+                "a", QXSM::onExit(&receiver, &Receiver::aExit));
+    QVERIFY(aExit);
+    QMetaObject::Connection a1Entry = stateMachine->connectToState("a1", &receiver,
+                                                                   QXSM::onEntry([&a1Entered]() {
+        a1Entered = true;
+    }));
+    QVERIFY(a1Entry);
+    QMetaObject::Connection a1Exit = stateMachine->connectToState("a1", &receiver,
+                                                                   QXSM::onExit([&a1Exited]() {
+        a1Exited = true;
+    }));
+    QVERIFY(a1Exit);
+
+    QMetaObject::Connection finalEntry = stateMachine->connectToState(
+                "final", QXSM::onEntry([&finalEntered]() {
+        finalEntered = true;
+    }));
+    QVERIFY(finalEntry);
+
+    QMetaObject::Connection finalExit = stateMachine->connectToState(
+                "final", QXSM::onExit([&finalExited]() {
+        finalExited = true;
+    }));
+    QVERIFY(finalExit);
+
+    stateMachine->start();
+
+    QTRY_VERIFY(a1Reached);
+    QTRY_VERIFY(finalReached);
+    QTRY_VERIFY(receiver.aReached);
+    QTRY_VERIFY(receiver.bReached);
+
+    QVERIFY(disconnect(a));
+    QVERIFY(disconnect(b));
+    QVERIFY(disconnect(a1));
+    QVERIFY(disconnect(final));
+
+#if defined(__cpp_return_type_deduction) && __cpp_return_type_deduction == 201304
+    QVERIFY(receiver.aEntered);
+    QVERIFY(!receiver.aExited);
+    QVERIFY(a1Entered);
+    QVERIFY(!a1Exited);
+    QVERIFY(finalEntered);
+    QVERIFY(!finalExited);
+
+    QVERIFY(disconnect(aEntry));
+    QVERIFY(disconnect(aExit));
+    QVERIFY(disconnect(a1Entry));
+    QVERIFY(disconnect(a1Exit));
+    QVERIFY(disconnect(finalEntry));
+    QVERIFY(disconnect(finalExit));
+#endif
+#endif
+}
+
+#if 0
+void tst_StateMachine::stateNames_data()
+{
+    QTest::addColumn<QString>("scxmlFileName");
+    QTest::addColumn<bool>("compressed");
+    QTest::addColumn<QStringList>("expectedStates");
+
+    QTest::newRow("stateNames-compressed") << QString(":/tst_statemachine/statenames.scxml")
+                                      << true
+                                      << (QStringList() << QString("a1") << QString("a2") << QString("final"));
+    QTest::newRow("stateNames-notCompressed") << QString(":/tst_statemachine/statenames.scxml")
+                                      << false
+                                      << (QStringList() << QString("top") << QString("a") << QString("a1") <<  QString("a2") << QString("b") << QString("final"));
+    QTest::newRow("stateNamesNested-compressed") << QString(":/tst_statemachine/statenamesnested.scxml")
+                                      << true
+                                      << (QStringList() << QString("a") << QString("b"));
+    QTest::newRow("stateNamesNested-notCompressed") << QString(":/tst_statemachine/statenamesnested.scxml")
+                                      << false
+                                      << (QStringList() << QString("super_top") << QString("a") << QString("b"));
+
+    QTest::newRow("ids1") << QString(":/tst_statemachine/ids1.scxml")
+                          << false
+                          << (QStringList() << QString("foo.bar") << QString("foo-bar")
+                              << QString("foo_bar") << QString("_"));
+}
+
+void tst_StateMachine::stateNames()
+{
+    QFETCH(QString, scxmlFileName);
+    QFETCH(bool, compressed);
+    QFETCH(QStringList, expectedStates);
+
+    QScopedPointer<QScxmlStateMachine> stateMachine(QScxmlStateMachine::fromFile(scxmlFileName));
+    QVERIFY(!stateMachine.isNull());
+    QCOMPARE(stateMachine->parseErrors().count(), 0);
+
+    QCOMPARE(stateMachine->stateNames(compressed), expectedStates);
+}
+
+void tst_StateMachine::activeStateNames_data()
+{
+    QTest::addColumn<QString>("scxmlFileName");
+    QTest::addColumn<bool>("compressed");
+    QTest::addColumn<QStringList>("expectedStates");
+
+    QTest::newRow("stateNames-compressed") << QString(":/tst_statemachine/statenames.scxml")
+                                      << true
+                                      << (QStringList() << QString("a1") << QString("final"));
+    QTest::newRow("stateNames-notCompressed") << QString(":/tst_statemachine/statenames.scxml")
+                                      << false
+                                      << (QStringList() << QString("top") << QString("a") << QString("a1") << QString("b") << QString("final"));
+    QTest::newRow("stateNamesNested-compressed") << QString(":/tst_statemachine/statenamesnested.scxml")
+                                      << true
+                                      << (QStringList() << QString("a") << QString("b"));
+    QTest::newRow("stateNamesNested-notCompressed") << QString(":/tst_statemachine/statenamesnested.scxml")
+                                      << false
+                                      << (QStringList() << QString("super_top") << QString("a") << QString("b"));
+}
+
+void tst_StateMachine::activeStateNames()
+{
+    QFETCH(QString, scxmlFileName);
+    QFETCH(bool, compressed);
+    QFETCH(QStringList, expectedStates);
+
+    QScopedPointer<QScxmlStateMachine> stateMachine(QScxmlStateMachine::fromFile(scxmlFileName));
+    QVERIFY(!stateMachine.isNull());
+
+    QSignalSpy stableStateSpy(stateMachine.data(), SIGNAL(reachedStableState()));
+
+    stateMachine->start();
+
+    stableStateSpy.wait(5000);
+
+    QCOMPARE(stateMachine->activeStateNames(compressed), expectedStates);
+}
+
+
+void tst_StateMachine::onExit()
+{
+#if defined(__cpp_return_type_deduction) && __cpp_return_type_deduction == 201304
+    // Test onExit being actually called
+
+    typedef QScxmlStateMachine QXSM;
+    QScopedPointer<QXSM> stateMachine(QXSM::fromFile(QString(":/tst_statemachine/eventoccurred.scxml")));
+
+    Receiver receiver;
+    bool aExited1 = false;
+
+    stateMachine->connectToState("a", QXSM::onExit([&aExited1]() { aExited1 = true; }));
+    stateMachine->connectToState("a", QXSM::onExit(&receiver, &Receiver::aExit));
+    stateMachine->connectToState("a", QXSM::onExit(&receiver, "aEnter"));
+    {
+        // Should not crash
+        Receiver receiver2;
+        stateMachine->connectToState("a", QXSM::onEntry(&receiver2, &Receiver::aEnter));
+        stateMachine->connectToState("a", QXSM::onEntry(&receiver2, "aExit"));
+        stateMachine->connectToState("a", QXSM::onExit(&receiver2, &Receiver::aExit));
+        stateMachine->connectToState("a", QXSM::onExit(&receiver2, "aEnter"));
+    }
+
+    stateMachine->start();
+    QTRY_VERIFY(receiver.aEntered);
+    QTRY_VERIFY(receiver.aExited);
+    QTRY_VERIFY(aExited1);
+#endif
+}
+
+bool hasChildEventRouters(QScxmlStateMachine *stateMachine)
+{
+    // Cast to QObject, to avoid ambigous "children" member.
+    const QObject &parentRouter = QScxmlStateMachinePrivate::get(stateMachine)->m_router;
+    return !parentRouter.children().isEmpty();
+}
+
+void tst_StateMachine::eventOccurred()
+{
+    QScopedPointer<QScxmlStateMachine> stateMachine(QScxmlStateMachine::fromFile(QString(":/tst_statemachine/eventoccurred.scxml")));
+    QVERIFY(!stateMachine.isNull());
+
+    qRegisterMetaType<QScxmlEvent>();
+    QSignalSpy finishedSpy(stateMachine.data(), SIGNAL(finished()));
+
+    int events = 0;
+    auto con1 = stateMachine->connectToEvent("internalEvent2", [&events](const QScxmlEvent &event) {
+        QCOMPARE(++events, 1);
+        QCOMPARE(event.name(), QString("internalEvent2"));
+        QCOMPARE(event.eventType(), QScxmlEvent::ExternalEvent);
+    });
+    QVERIFY(con1);
+
+    auto con2 = stateMachine->connectToEvent("externalEvent", [&events](const QScxmlEvent &event) {
+        QCOMPARE(++events, 2);
+        QCOMPARE(event.name(), QString("externalEvent"));
+        QCOMPARE(event.eventType(), QScxmlEvent::ExternalEvent);
+    });
+    QVERIFY(con2);
+
+    auto con3 = stateMachine->connectToEvent("timeout", [&events](const QScxmlEvent &event) {
+        QCOMPARE(++events, 3);
+        QCOMPARE(event.name(), QString("timeout"));
+        QCOMPARE(event.eventType(), QScxmlEvent::ExternalEvent);
+    });
+    QVERIFY(con3);
+
+    auto con4 = stateMachine->connectToEvent("done.*", [&events](const QScxmlEvent &event) {
+        QCOMPARE(++events, 4);
+        QCOMPARE(event.name(), QString("done.state.top"));
+        QCOMPARE(event.eventType(), QScxmlEvent::ExternalEvent);
+    });
+    QVERIFY(con4);
+
+    auto con5 = stateMachine->connectToEvent("done.state", [&events](const QScxmlEvent &event) {
+        QCOMPARE(++events, 5);
+        QCOMPARE(event.name(), QString("done.state.top"));
+        QCOMPARE(event.eventType(), QScxmlEvent::ExternalEvent);
+    });
+    QVERIFY(con5);
+
+    auto con6 = stateMachine->connectToEvent("done.state.top", [&events](const QScxmlEvent &event) {
+        QCOMPARE(++events, 6);
+        QCOMPARE(event.name(), QString("done.state.top"));
+        QCOMPARE(event.eventType(), QScxmlEvent::ExternalEvent);
+    });
+    QVERIFY(con6);
+
+    stateMachine->start();
+
+    finishedSpy.wait(5000);
+    QCOMPARE(events, 6);
+
+    QVERIFY(disconnect(con1));
+    QVERIFY(disconnect(con2));
+    QVERIFY(disconnect(con3));
+    QVERIFY(disconnect(con4));
+    QVERIFY(disconnect(con5));
+    QVERIFY(disconnect(con6));
+
+    QTRY_VERIFY(!hasChildEventRouters(stateMachine.data()));
+}
+
+void tst_StateMachine::doneDotStateEvent()
+{
+    QScopedPointer<QScxmlStateMachine> stateMachine(QScxmlStateMachine::fromFile(QString(":/tst_statemachine/stateDotDoneEvent.scxml")));
+    QVERIFY(!stateMachine.isNull());
+
+    QSignalSpy finishedSpy(stateMachine.data(), SIGNAL(finished()));
+
+    stateMachine->start();
+    finishedSpy.wait(5000);
+    QCOMPARE(finishedSpy.count(), 1);
+    QCOMPARE(stateMachine->activeStateNames(true).size(), 1);
+    qDebug() << stateMachine->activeStateNames(true);
+    QVERIFY(stateMachine->activeStateNames(true).contains(QLatin1String("success")));
+}
+
+void tst_StateMachine::running()
+{
+    QScopedPointer<QScxmlStateMachine> stateMachine(
+                QScxmlStateMachine::fromFile(QString(":/tst_statemachine/statenames.scxml")));
+    QVERIFY(!stateMachine.isNull());
+
+    QSignalSpy runningChangedSpy(stateMachine.data(), SIGNAL(runningChanged(bool)));
+
+    QCOMPARE(stateMachine->isRunning(), false);
+
+    stateMachine->start();
+
+    QCOMPARE(runningChangedSpy.count(), 1);
+    QCOMPARE(stateMachine->isRunning(), true);
+
+    stateMachine->stop();
+
+    QCOMPARE(runningChangedSpy.count(), 2);
+    QCOMPARE(stateMachine->isRunning(), false);
+}
+#endif
+QTEST_MAIN(StatesPerSecond)
+
+#include "StatesPerSecond.moc"
+
+
diff --git a/contrib/benchmarks/qt/statemachine.pro b/contrib/benchmarks/qt/statemachine.pro
new file mode 100644
index 0000000..d3eae77
--- /dev/null
+++ b/contrib/benchmarks/qt/statemachine.pro
@@ -0,0 +1,11 @@
+QT = core gui qml testlib scxml
+CONFIG += testcase
+
+TARGET = StatesPerSecond
+CONFIG += console
+CONFIG -= app_bundle
+
+TEMPLATE = app
+
+SOURCES += \
+    StatesPerSecond.cpp
diff --git a/contrib/benchmarks/run.sh b/contrib/benchmarks/run.sh
new file mode 100755
index 0000000..ac13779
--- /dev/null
+++ b/contrib/benchmarks/run.sh
@@ -0,0 +1,347 @@
+#!/bin/bash
+ME=`basename $0`
+DIR="$( cd "$( dirname "$0" )" && pwd )"
+CWD=`pwd`
+cd ${DIR}
+
+TIMEOUT=25s
+
+RUN_QT=false
+RUN_APACHE=false
+RUN_SCION=false
+RUN_SCXMLCC=false
+RUN_USCXML=false
+RUN_LXSC=false
+
+PLOT=false
+
+PATTERN='*.scxml'
+
+if [ ! -d "./logs" ]; then
+	mkdir logs
+fi
+
+# https://superuser.com/a/473715
+function canonicalPath
+{
+    local path="$1" ; shift
+    if [ -d "$path" ]
+    then
+        echo "$(cd "$path" ; pwd)"
+    else
+        local b=$(basename "$path")
+        local p=$(dirname "$path")
+        echo "$(cd "$p" ; pwd)/$b"
+    fi
+}
+
+# ===== qt ================
+function init-qt {
+	cd qt
+	if [ ! -x "./build/StatesPerSecond" ]; then
+		mkdir build
+		cd build
+		qmake ..
+		make
+		cd ..
+	fi
+	cd ..
+}
+
+function run-qt {
+	BENCHMARK=$1 
+	SC_NAME=$2
+	export USCXML_BENCHMARK=${BENCHMARK}
+
+	cd qt/build
+	make
+	timeout ${TIMEOUT} ./StatesPerSecond |tee ../../logs/${SC_NAME}-qt.log
+	cd ../..
+}
+
+function clean-qt {
+	cd qt
+	rm -rf build
+	cd ..
+}
+
+# ===== APACHE ================
+function init-apache {
+	cd apache
+	mvn compile
+	cd ..
+}
+
+function run-apache {
+	BENCHMARK=$1 
+	SC_NAME=$2
+	export USCXML_BENCHMARK=${BENCHMARK}
+
+	cd apache
+	timeout ${TIMEOUT} mvn test |tee ../logs/${SC_NAME}-apache.log
+	cd ..
+
+}
+
+function clean-apache {
+	cd apache
+	rm -rf target
+	cd ..
+}
+
+
+# ===== USCXML ================
+function init-uscxml {
+	cd uscxml
+	if [ ! -x "./statesPerSecond" ]; then
+		g++ -std=c++11 \
+		./statesPerSecond.cpp \
+		-I/usr/local/include \
+		-I../../../build/cli/deps/xerces-c/include/ \
+		-luscxml \
+		-L ../../../build/cli/deps/xerces-c/lib/ \
+		-lxerces-c \
+		-o statesPerSecond
+	fi
+	cd ..
+}
+
+function run-uscxml {
+	BENCHMARK=$1 
+	SC_NAME=$2
+
+	cd uscxml
+	timeout 600s ./statesPerSecond ${BENCHMARK} fast
+	timeout ${TIMEOUT} ./statesPerSecond ${BENCHMARK} fast |tee ../logs/${SC_NAME}-uscxml-fast.log
+	USCXML_NOCACHE_FILES=YES \
+	timeout ${TIMEOUT} ./statesPerSecond ${BENCHMARK} large |tee ../logs/${SC_NAME}-uscxml-large.log
+	cd ..
+}
+
+function clean-uscxml {
+	cd uscxml
+	rm statesPerSecond
+	cd ..
+
+}
+
+# ===== LXSC ================
+function init-lxsc {
+	cd lxsc
+
+	if [ ! -d "./lxsc" ]; then
+		git clone https://github.com/Phrogz/LXSC.git
+		# increase microstep sequence considerably
+		sed -i '.bak' 's/S\.MAX_ITERATIONS\ =\ 1000/S\.MAX_ITERATIONS\ =\ 1000000/' ./LXSC/lib/runtime.lua
+	fi
+
+	cd ..
+}
+
+function run-lxsc {
+	BENCHMARK=$1 
+	SC_NAME=$2
+
+	cd lxsc
+	timeout ${TIMEOUT} lua ./statesPerSecond.lua ${BENCHMARK} |tee ../logs/${SC_NAME}-lxsc.log
+	cd ..
+}
+
+function clean-lxsc {
+	cd lxsc
+	rm -rf LXSC
+	cd ..
+}
+
+# ===== SCION ================
+function init-scion {
+	cd scion
+	if [ ! -d "./node_modules" ]; then
+		npm install scxml babel-polyfill
+	fi
+	cd ..
+}
+
+function run-scion {
+	BENCHMARK=$1 
+	SC_NAME=$2
+
+	cd scion
+
+	timeout ${TIMEOUT} node ./statesPerSecond.js ${BENCHMARK} |tee ../logs/${SC_NAME}-scion.log
+
+	cd ..
+}
+
+function clean-scion {
+	cd scion
+	rm -rf node_modules
+	cd ..
+}
+
+# ===== SCXMLCC ================
+function init-scxmlcc {
+	cd scxmlcc
+
+	# check out from git
+	if [ ! -d "./scxmlcc" ]; then
+		git clone https://github.com/jp-embedded/scxmlcc.git
+	fi
+
+	# compile transpiler
+	if [ ! -x "./scxmlcc/src/scxmlcc" ]; then
+		export CXXFLAGS=-I/opt/local/include
+		cp ./makefile scxmlcc/src
+		cd scxmlcc/src
+		touch version_auto.h 
+		make
+		cd ../..
+	fi
+
+	cd ..
+}
+
+function run-scxmlcc {
+	BENCHMARK=$1 
+	SC_NAME=$2
+
+	cd scxmlcc
+	rm test
+	timeout 600s ./scxmlcc/src/scxmlcc -i ${BENCHMARK} -o ./test.h
+	timeout ${TIMEOUT} g++ -DMACHINE_NAME=sc_benchmark ./statesPerSecond.cpp -o test
+	timeout ${TIMEOUT} ./test |tee ../logs/${SC_NAME}-scxmlcc.log
+
+	cd ..
+}
+
+function clean-scxmlcc {
+	cd scxmlcc
+	rm -rf logs
+	rm -rf scxmlcc
+	rm test
+	rm test.h
+	cd ..
+}  
+
+
+# =======================================
+while [[ $# -gt 0 ]]
+do
+key="$1"
+case $key in
+		-p|--pattern)
+		PATTERN="$2"
+		;;
+
+		-p|--plot)
+		
+		;;
+		-c|--clean)
+		clean-scxmlcc
+		clean-scion
+		clean-lxsc
+		clean-uscxml
+		clean-apache
+		clean-qt
+		rm -rf logs
+		exit
+		;;
+
+		--all)
+		init-scxmlcc
+		init-uscxml
+		init-lxsc
+		init-scion
+		init-apache
+		init-qt
+		RUN_QT=true
+		RUN_APACHE=true
+		RUN_SCXMLCC=true
+		RUN_USCXML=true
+		RUN_LXSC=true
+		RUN_SCION=true
+		;;
+
+		--scxmlcc)
+		init-scxmlcc
+		RUN_SCXMLCC=true
+		;;
+
+		--uscxml)
+		init-uscxml
+		RUN_USCXML=true
+		;;
+
+		--lxsc)
+		init-lxsc
+		RUN_LXSC=true
+		;;
+
+		--qt)
+		init-qt
+		RUN_QT=true
+		;;
+
+		--scion)
+		init-scion
+		RUN_SCION=true
+		;;
+
+		--apache)
+		init-apache
+		RUN_APACHE=true
+		;;
+
+		*)
+		;;
+esac
+shift # past argument or value
+done
+
+BENCHMARKS=`find $(pwd)/../../test/benchmarks -type f -name ${PATTERN}`
+
+for BENCHMARK in $BENCHMARKS
+do
+	BENCHMARK=$(canonicalPath $BENCHMARK)
+
+	SC_NAME=$(basename "$BENCHMARK" .scxml)
+
+	echo "== Running ${SC_NAME} from ${BENCHMARK}"
+	
+	if [ "$RUN_SCION" = true ] ; then
+		echo "==== with SCION"
+		run-scion ${BENCHMARK} ${SC_NAME}
+	fi 
+
+	if [ "$RUN_SCXMLCC" = true ] ; then
+		echo "==== with SCXMLCC"
+		run-scxmlcc ${BENCHMARK} ${SC_NAME}
+	fi 
+
+	if [ "$RUN_LXSC" = true ] ; then
+		echo "==== with LXSC"
+		run-lxsc ${BENCHMARK} ${SC_NAME}
+	fi 
+
+	if [ "$RUN_QT" = true ] ; then
+		echo "==== with QT"
+		run-qt ${BENCHMARK} ${SC_NAME}
+	fi 
+
+	if [ "$RUN_USCXML" = true ] ; then
+		echo "==== with USCXML"
+		run-uscxml ${BENCHMARK} ${SC_NAME}
+	fi 
+
+	if [ "$RUN_APACHE" = true ] ; then
+		echo "==== with APACHE"
+		run-apache ${BENCHMARK} ${SC_NAME}
+	fi 
+
+done
+sed -i '.bak' '/^[0-9\.]*, [0-9\.]*/!d' ./logs/*.log
+rm -f logs/*.bak
+
+if [ "$PLOT" = true ] ; then
+	./plot.sh
+fi
diff --git a/contrib/benchmarks/scion/statesPerSecond.js b/contrib/benchmarks/scion/statesPerSecond.js
new file mode 100644
index 0000000..2a15fcb
--- /dev/null
+++ b/contrib/benchmarks/scion/statesPerSecond.js
@@ -0,0 +1,50 @@
+require('babel-polyfill');
+let scxml = require('scxml');
+
+function nowInMS() {
+	var hrTime = process.hrtime();
+	return hrTime[0] * 1000 + hrTime[1] / 1000000
+}
+
+var args = process.argv.splice(process.execArgv.length + 2);
+var started = nowInMS();
+var initTimeMs;
+
+scxml.pathToModel(args[0], function(err,model){
+  if(err) throw err;
+
+  model.prepare(function(err, fnModel) {
+
+  	var iterations = 0;
+  	var mark = nowInMS();
+
+    if(err) throw err;
+
+    //instantiate the interpreter
+    var sc = new scxml.scion.Statechart(fnModel);
+
+		initTimeMs = nowInMS() - started;
+
+		sc.registerListener({onEntry : function(stateId) {
+				if (stateId == "mark") {
+					iterations++;
+
+					var now = nowInMS();
+					if (now - mark > 1000) {
+						console.log(initTimeMs + ", " + iterations);
+						mark = now;
+						iterations = 0;
+					}
+
+				}
+			}
+		});
+
+    //start the interpreter
+    sc.start();
+
+    //send the init event
+    sc.gen({name:"init",data:null});
+
+  });
+})
\ No newline at end of file
diff --git a/contrib/benchmarks/scxmlcc/makefile b/contrib/benchmarks/scxmlcc/makefile
new file mode 100644
index 0000000..0466459
--- /dev/null
+++ b/contrib/benchmarks/scxmlcc/makefile
@@ -0,0 +1,35 @@
+#*************************************************************************
+#** Copyright (C) 2013 Jan Pedersen <jp@jp-embedded.com>
+#** 
+#** This program is free software: you can redistribute it and/or modify
+#** it under the terms of the GNU General Public License as published by
+#** the Free Software Foundation, either version 3 of the License, or
+#** (at your option) any later version.
+#** 
+#** This program is distributed in the hope that it will be useful,
+#** but WITHOUT ANY WARRANTY; without even the implied warranty of
+#** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#** GNU General Public License for more details.
+#** 
+#** You should have received a copy of the GNU General Public License
+#** along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#*************************************************************************
+
+OBJS := main.o cpp_output.o scxml_parser.o version.o
+CPPFLAGS := -Wall -MD -MP -O2
+
+all: scxmlcc
+
+scxmlcc: $(OBJS)
+	$(CXX) -o $@ $^ -L/opt/local/lib -lboost_program_options-mt -lboost_system-mt -lboost_filesystem-mt
+
+clean:
+	rm -f $(OBJS) $(OBJS:.o=.d) scxmlcc autorevision.mk version_auto.h
+
+autorevision.mk:
+	autorevision -tsh > $@ && sh makerevision.sh || truncate -s0 version_auto.h
+.PHONY: autorevision.mk
+-include autorevision.mk	
+
+-include $(OBJS:.o=.d)
+
diff --git a/contrib/benchmarks/scxmlcc/statesPerSecond.cpp b/contrib/benchmarks/scxmlcc/statesPerSecond.cpp
new file mode 100644
index 0000000..e0dc7b2
--- /dev/null
+++ b/contrib/benchmarks/scxmlcc/statesPerSecond.cpp
@@ -0,0 +1,47 @@
+#include <iostream>
+#include <chrono>
+#include <stdlib.h>
+
+#include "test.h"
+
+using namespace std;
+using namespace std::chrono;
+
+typedef MACHINE_NAME sc;
+
+long iterations = 0;
+long initMs = 0;
+system_clock::time_point report;
+system_clock::time_point endTime;
+
+template<> void sc::state_actions<sc::state_mark>::enter(sc::data_model &m)
+{
+	iterations++;
+	system_clock::time_point now = system_clock::now();
+
+	if (now > report) {
+		report = now + seconds(1);
+		std::cout << initMs << ", " << iterations << std::endl;
+		iterations = 0;
+	}
+	if (now > endTime) {
+		::exit(EXIT_SUCCESS);
+	}
+}
+
+int main(int argc, char *argv[])
+{
+	system_clock::time_point start = system_clock::now();
+
+	sc sc0;
+	system_clock::time_point now = system_clock::now();
+
+	initMs = duration_cast<milliseconds>(now - start).count();
+
+	start = now;
+	report = start + seconds(1);
+	endTime = start + seconds(10);
+
+	sc0.init();
+	return 0;
+}
diff --git a/contrib/benchmarks/uscxml/statesPerSecond.cpp b/contrib/benchmarks/uscxml/statesPerSecond.cpp
new file mode 100644
index 0000000..f6e62e6
--- /dev/null
+++ b/contrib/benchmarks/uscxml/statesPerSecond.cpp
@@ -0,0 +1,69 @@
+#include <iostream>
+#include <chrono>
+#include <stdlib.h>
+
+#include "uscxml/uscxml.h"
+#include "uscxml/interpreter/InterpreterMonitor.h"
+#include "uscxml/util/DOM.h"
+
+using namespace std;
+using namespace uscxml;
+using namespace std::chrono;
+
+long iterations = 0;
+long initMs = 0;
+system_clock::time_point now;
+system_clock::time_point report;
+system_clock::time_point endTime;
+
+class PerfMon : public InterpreterMonitor {
+	public: 
+	virtual void beforeEnteringState(Interpreter& interpreter, const XERCESC_NS::DOMElement* state) {
+		if (HAS_ATTR(state, X("id")) && ATTR(state, X("id")) == "mark") {
+			iterations++;
+			now = system_clock::now();
+			if (now > report) {
+				report = now + seconds(1);
+				std::cout << initMs << ", " << iterations << std::endl;
+				iterations = 0;
+			}
+		}
+		if (now > endTime) {
+			::exit(EXIT_SUCCESS);
+		}
+	}
+};
+
+int main(int argc, char *argv[])
+{
+	system_clock::time_point start = system_clock::now();
+
+	Interpreter sc = Interpreter::fromURL(argv[1]);
+	ActionLanguage al;
+	if (argc > 2) {
+		if (std::string(argv[2]) == "large") {
+	    al.microStepper = Factory::getInstance()->createMicroStepper("large", (MicroStepCallbacks*)sc);
+		} else {
+  	  al.microStepper = Factory::getInstance()->createMicroStepper("fast", (MicroStepCallbacks*)sc);
+		}
+  }
+	sc.setActionLanguage(al);
+
+	InterpreterState state = sc.step(); // initialize?
+
+	PerfMon mon;
+	sc.addMonitor(&mon);
+
+	now = system_clock::now();
+	initMs = duration_cast<milliseconds>(now - start).count();
+
+	start = now;
+	report = start + seconds(1);
+	endTime = start + seconds(10);
+
+	while(true) {
+		sc.step();
+	}
+
+	return 0;
+}