Add a feature to QTestLib to correct benchmark results.

Two special values are added to the measurement of benchmarks: a zero
and a comparison. The "zero" value is to be used when the test incurs
a non-negligible overhead just to run. The benchmark author should add
an "empty" run that measures basically the overhead and set it to
attribute QTest::Zero.

The "zero" value will be subtracted from any benchmark measurements
before reporting (currently the plain logger only). Note that all
benchmark types don't make sense, notably those already measurement
some kind of rate, like frames per second and bytes per second.

The "baseline" value is the comparison (with the "zero"
subtracted"). When the baseline is set, the library will instead
report the ratio between the benchmark and the baseline, in percentage
or a multiple (e.g. "75%" [of the time] or "10.0x" [faster]).

Adding measurements in "robes" left for future improvement.

This feature is not reviewed and will be reverted later.

20 files changed, 134 insertions, 33 deletions

diff --git a/src/testlib/qabstracttestlogger_p.h b/src/testlib/qabstracttestlogger_p.h
index d116a6e..50f651a 100644
--- a/src/testlib/qabstracttestlogger_p.h
+++ b/src/testlib/qabstracttestlogger_p.h
@@ -90,7 +90,7 @@ public:
 
     virtual void addIncident(IncidentTypes type, const char *description,
                              const char *file = 0, int line = 0) = 0;
-    virtual void addBenchmarkResult(const QBenchmarkResult &result) = 0;
+    virtual void addBenchmarkResult(const QBenchmarkResult &result, const QBenchmarkResult &correctedResult) = 0;
 
     virtual void addMessage(MessageTypes type, const char *message,
                             const char *file = 0, int line = 0) = 0;
diff --git a/src/testlib/qbenchmark.cpp b/src/testlib/qbenchmark.cpp
index d933fb1..cb56154 100644
--- a/src/testlib/qbenchmark.cpp
+++ b/src/testlib/qbenchmark.cpp
@@ -181,6 +181,12 @@ void QBenchmarkTestMethodData::setResult(
         QBenchmarkGlobalData::current->context, value, iterationCount, metric, setByMacro);
 }
 
+void QBenchmarkTestMethodData::clearSpecialResults()
+{
+    for (int i = 0; i < QTest::BenchmarkSpecialCount; ++i)
+        specialResults[i] = QBenchmarkResult();
+}
+
 /*!
     \class QTest::QBenchmarkIterationController
     \internal
diff --git a/src/testlib/qbenchmark_p.h b/src/testlib/qbenchmark_p.h
index ace17db..387ad8b 100644
--- a/src/testlib/qbenchmark_p.h
+++ b/src/testlib/qbenchmark_p.h
@@ -177,8 +177,10 @@ public:
     bool resultsAccepted() const { return resultAccepted; }
     int adjustIterationCount(int suggestion);
     void setResult(qreal value, QTest::QBenchmarkMetric metric, bool setByMacro = true);
+    void clearSpecialResults();
 
     QBenchmarkResult result;
+    QBenchmarkResult specialResults[QTest::BenchmarkSpecialCount];
     bool resultAccepted;
     bool runOnce;
     int iterationCount;
diff --git a/src/testlib/qbenchmarkmetric.cpp b/src/testlib/qbenchmarkmetric.cpp
index 025bf46..1631e98 100644
--- a/src/testlib/qbenchmarkmetric.cpp
+++ b/src/testlib/qbenchmarkmetric.cpp
@@ -81,6 +81,8 @@ const char * QTest::benchmarkMetricName(QBenchmarkMetric metric)
         return "InstructionReads";
     case Events:
         return "Events";
+    case Percentage:
+        return "Percentage";
     default:
         return "";
     }
@@ -108,6 +110,7 @@ const char * QTest::benchmarkMetricUnit(QBenchmarkMetric metric)
         return "instruction reads";
     case Events:
         return "events";
+    case Percentage:
     default:
         return "";
     }
diff --git a/src/testlib/qbenchmarkmetric.h b/src/testlib/qbenchmarkmetric.h
index c8ab2fd..a413108 100644
--- a/src/testlib/qbenchmarkmetric.h
+++ b/src/testlib/qbenchmarkmetric.h
@@ -59,7 +59,8 @@ enum QBenchmarkMetric {
     WalltimeMilliseconds,
     CPUTicks,
     InstructionReads,
-    Events
+    Events,
+    Percentage
 };
 
 }
diff --git a/src/testlib/qplaintestlogger.cpp b/src/testlib/qplaintestlogger.cpp
index e1ae534..c5c6fd6 100644
--- a/src/testlib/qplaintestlogger.cpp
+++ b/src/testlib/qplaintestlogger.cpp
@@ -315,7 +315,7 @@ namespace QTest {
     }
 
 //    static void printBenchmarkResult(const char *bmtag, int value, int iterations)
-    static void printBenchmarkResult(const QBenchmarkResult &result)
+    static void printBenchmarkResult(const QBenchmarkResult &result, const QBenchmarkResult &corrected)
     {
         const char *bmtag = QTest::benchmarkResult2String();
 
@@ -333,7 +333,6 @@ namespace QTest {
             QTest::qt_snprintf(bufTag, sizeof(bufTag), ":\"%s\"", tag.data());
         }
 
-
         char fillFormat[8];
         int fillLength = 5;
         QTest::qt_snprintf(
@@ -341,6 +340,30 @@ namespace QTest {
         char fill[1024];
         QTest::qt_snprintf(fill, sizeof(fill), fillFormat, "");
 
+        char buf1_[1024];
+        buf1_[0] = 0;
+        if (corrected.valid) {
+            if (corrected.metric == QTest::Percentage) {
+                if (corrected.value <= 3 && corrected.value >= -1)
+                    QTest::qt_snprintf(
+                        buf1_, sizeof(buf1_), "%.1f%% (of baseline), actual: ", corrected.value * 100);
+                else
+                    QTest::qt_snprintf(
+                        buf1_, sizeof(buf1_), "%.2fx (of baseline), actual: ", corrected.value);
+            } else {
+                const char * unitText = QTest::benchmarkMetricUnit(corrected.metric);
+
+                qreal valuePerIteration = qreal(corrected.value) / qreal(corrected.iterations);
+                char resultBuffer[100] = "";
+                formatResult(resultBuffer, 100, valuePerIteration, countSignificantDigits(corrected.value));
+
+                QTest::qt_snprintf(
+                    buf1_, sizeof(buf1_), "%s %s (corrected), actual: ",
+                    resultBuffer,
+                    unitText);
+            }
+        }
+
         const char * unitText = QTest::benchmarkMetricUnit(result.metric);
 
         qreal valuePerIteration = qreal(result.value) / qreal(result.iterations);
@@ -349,18 +372,10 @@ namespace QTest {
 
         char buf2[1024];
         QTest::qt_snprintf(
-            buf2, sizeof(buf2), "%s %s",
+            buf2, sizeof(buf2), "%s %s per iteration",
             resultBuffer,
             unitText);
 
-        char buf2_[1024];
-        QByteArray iterationText = " per iteration";
-        Q_ASSERT(result.iterations > 0);
-        QTest::qt_snprintf(
-            buf2_,
-            sizeof(buf2_), "%s",
-            iterationText.data());
-
         char buf3[1024];
         Q_ASSERT(result.iterations > 0);
         formatResult(resultBuffer, 100, result.value, countSignificantDigits(result.value));
@@ -373,9 +388,9 @@ namespace QTest {
 
         if (result.setByMacro) {
             QTest::qt_snprintf(
-                buf, sizeof(buf), "%s%s%s%s%s%s\n", buf1, bufTag, fill, buf2, buf2_, buf3);
+                buf, sizeof(buf), "%s%s%s%s%s%s\n", buf1, bufTag, fill, buf1_, buf2, buf3);
         } else {
-            QTest::qt_snprintf(buf, sizeof(buf), "%s%s%s%s\n", buf1, bufTag, fill, buf2);
+            QTest::qt_snprintf(buf, sizeof(buf), "%s%s%s%s%s\n", buf1, bufTag, fill, buf1_, buf2);
         }
 
         memcpy(buf, bmtag, strlen(bmtag));
@@ -471,10 +486,10 @@ void QPlainTestLogger::addIncident(IncidentTypes type, const char *description,
     QTest::printMessage(QTest::incidentType2String(type), description, file, line);
 }
 
-void QPlainTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
+void QPlainTestLogger::addBenchmarkResult(const QBenchmarkResult &result, const QBenchmarkResult &corrected)
 {
 //    QTest::printBenchmarkResult(QTest::benchmarkResult2String(), value, iterations);
-    QTest::printBenchmarkResult(result);
+    QTest::printBenchmarkResult(result, corrected);
 }
 
 void QPlainTestLogger::addMessage(MessageTypes type, const char *message,
diff --git a/src/testlib/qplaintestlogger_p.h b/src/testlib/qplaintestlogger_p.h
index 054ec1e..c5165dd 100644
--- a/src/testlib/qplaintestlogger_p.h
+++ b/src/testlib/qplaintestlogger_p.h
@@ -71,7 +71,7 @@ public:
 
     void addIncident(IncidentTypes type, const char *description,
                      const char *file = 0, int line = 0);
-    void addBenchmarkResult(const QBenchmarkResult &result);
+    void addBenchmarkResult(const QBenchmarkResult &result, const QBenchmarkResult &corrected);
 
     void addMessage(MessageTypes type, const char *message,
                     const char *file = 0, int line = 0);
diff --git a/src/testlib/qtest_global.h b/src/testlib/qtest_global.h
index 28c8617..2e7217c 100644
--- a/src/testlib/qtest_global.h
+++ b/src/testlib/qtest_global.h
@@ -81,6 +81,15 @@ namespace QTest
 {
     enum SkipMode { SkipSingle = 1, SkipAll = 2 };
     enum TestFailMode { Abort = 1, Continue = 2 };
+    enum BenchmarkDataMode {
+        Normal = -1,
+        Zero = 0,
+        Subtract = Zero,
+        Baseline,
+        Divide = Baseline,
+
+        BenchmarkSpecialCount = 2
+    };
 
     int Q_TESTLIB_EXPORT qt_snprintf(char *str, int size, const char *format, ...);
 }
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index 287d8e6..698d9b5 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -1384,8 +1384,17 @@ static void qInvokeTestMethodDataEntry(char *slot)
              && (++i < QBenchmarkGlobalData::current->adjustMedianIterationCount()));
 
     if (QBenchmarkTestMethodData::current->isBenchmark()
-        && QBenchmarkTestMethodData::current->resultsAccepted())
-        QTestLog::addBenchmarkResult(qMedian(results));
+        && QBenchmarkTestMethodData::current->resultsAccepted()) {
+        QBenchmarkResult result = qMedian(results);
+
+        QBenchmarkResult *specialResults = QBenchmarkTestMethodData::current->specialResults;
+        if (QTestResult::currentTestData()) {
+            QTest::BenchmarkDataMode benchMode = QTestResult::currentTestData()->benchmarkSpecialData();
+            if (benchMode > QTest::Normal && benchMode < QTest::BenchmarkSpecialCount)
+                specialResults[benchMode] = result;
+        }
+        QTestLog::addBenchmarkResult(result, specialResults);
+    }
 }
 
 /*!
@@ -1453,6 +1462,8 @@ static bool qInvokeTestMethod(const char *slotName, const char *data=0)
                 }
             }
 
+            benchmarkData.clearSpecialResults();
+
             /* For each entry in the data table, do: */
             do {
                 if (!data || !qstrcmp(data, table.testData(curDataIndex)->dataTag())) {
@@ -2038,10 +2049,15 @@ void QTest::addColumnInternal(int id, const char *name)
 */
 QTestData &QTest::newRow(const char *dataTag)
 {
+    return newRow(dataTag, Normal);
+}
+
+QTestData &QTest::newRow(const char *dataTag, BenchmarkDataMode mode)
+{
     QTestTable *tbl = QTestTable::currentTestTable();
     QTEST_ASSERT_X(tbl, "QTest::addColumn()", "Cannot add testdata outside of a _data slot.");
 
-    return *tbl->newData(dataTag);
+    return *tbl->newData(dataTag, mode);
 }
 
 /*! \fn void QTest::addColumn(const char *name, T *dummy = 0)
diff --git a/src/testlib/qtestcase.h b/src/testlib/qtestcase.h
index a2cc26d..c0787a8 100644
--- a/src/testlib/qtestcase.h
+++ b/src/testlib/qtestcase.h
@@ -125,7 +125,6 @@ namespace QTest
         return 0;
     }
 
-
     Q_TESTLIB_EXPORT char *toHexRepresentation(const char *ba, int length);
     Q_TESTLIB_EXPORT char *toString(const char *);
     Q_TESTLIB_EXPORT char *toString(const void *);
@@ -168,6 +167,7 @@ namespace QTest
         addColumnInternal(qMetaTypeId<T>(), name);
     }
     Q_TESTLIB_EXPORT QTestData &newRow(const char *dataTag);
+    Q_TESTLIB_EXPORT QTestData &newRow(const char *dataTag, BenchmarkDataMode mode);
 
     template <typename T>
     inline bool qCompare(T const &t1, T const &t2, const char *actual, const char *expected,
diff --git a/src/testlib/qtestdata.cpp b/src/testlib/qtestdata.cpp
index 588ad64..c22dbe2 100644
--- a/src/testlib/qtestdata.cpp
+++ b/src/testlib/qtestdata.cpp
@@ -53,12 +53,13 @@ QT_BEGIN_NAMESPACE
 class QTestDataPrivate
 {
 public:
-    QTestDataPrivate(): tag(0), parent(0), data(0), dataCount(0) {}
+    QTestDataPrivate(): tag(0), parent(0), data(0), dataCount(0), benchmarkDataMode(QTest::Normal) {}
 
     char *tag;
     QTestTable *parent;
     void **data;
     int dataCount;
+    QTest::BenchmarkDataMode benchmarkDataMode;
 };
 
 QTestData::QTestData(const char *tag, QTestTable *parent)
@@ -72,6 +73,18 @@ QTestData::QTestData(const char *tag, QTestTable *parent)
     memset(d->data, 0, parent->elementCount() * sizeof(void*));
 }
 
+QTestData::QTestData(const char *tag, QTest::BenchmarkDataMode benchMode, QTestTable *parent)
+{
+    QTEST_ASSERT(tag);
+    QTEST_ASSERT(parent);
+    d = new QTestDataPrivate;
+    d->tag = qstrdup(tag);
+    d->parent = parent;
+    d->data = new void *[parent->elementCount()];
+    d->benchmarkDataMode = benchMode;
+    memset(d->data, 0, parent->elementCount() * sizeof(void*));
+}
+
 QTestData::~QTestData()
 {
     for (int i = 0; i < d->dataCount; ++i) {
@@ -119,4 +132,9 @@ int QTestData::dataCount() const
     return d->dataCount;
 }
 
+QTest::BenchmarkDataMode QTestData::benchmarkSpecialData() const
+{
+    return d->benchmarkDataMode;
+}
+
 QT_END_NAMESPACE
diff --git a/src/testlib/qtestdata.h b/src/testlib/qtestdata.h
index b39bce2..f154b8f 100644
--- a/src/testlib/qtestdata.h
+++ b/src/testlib/qtestdata.h
@@ -66,10 +66,12 @@ public:
     const char *dataTag() const;
     QTestTable *parent() const;
     int dataCount() const;
+    QTest::BenchmarkDataMode benchmarkSpecialData() const;
 
 private:
     friend class QTestTable;
     QTestData(const char *tag = 0, QTestTable *parent = 0);
+    QTestData(const char *tag, QTest::BenchmarkDataMode mode, QTestTable *parent = 0);
 
     Q_DISABLE_COPY(QTestData)
 
diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp
index ed9a005..28358be 100644
--- a/src/testlib/qtestlog.cpp
+++ b/src/testlib/qtestlog.cpp
@@ -46,6 +46,7 @@
 #include "QtTest/private/qabstracttestlogger_p.h"
 #include "QtTest/private/qplaintestlogger_p.h"
 #include "QtTest/private/qxmltestlogger_p.h"
+#include "QtTest/private/qbenchmark_p.h"
 #include <QtCore/qatomic.h>
 #include <QtCore/qbytearray.h>
 
@@ -285,10 +286,38 @@ void QTestLog::addSkip(const char *msg, QTest::SkipMode /*mode*/,
     QTest::testLogger->addMessage(QAbstractTestLogger::Skip, msg, file, line);
 }
 
-void QTestLog::addBenchmarkResult(const QBenchmarkResult &result)
+void QTestLog::addBenchmarkResult(const QBenchmarkResult &result, const QBenchmarkResult *specialResults)
 {
     QTEST_ASSERT(QTest::testLogger);
-    QTest::testLogger->addBenchmarkResult(result);
+    QTEST_ASSERT(specialResults);
+
+    QBenchmarkResult corrected;
+    const QBenchmarkResult &zero = specialResults[QTest::Zero];
+    if (zero.valid && zero.metric == result.metric) {
+        // subtract the zero result
+        corrected = result;
+        if (zero.iterations == corrected.iterations)
+            corrected.value -= zero.value;
+        else
+            corrected.value -= zero.value / zero.iterations * corrected.iterations;
+    }
+
+    const QBenchmarkResult &baseline = specialResults[QTest::Baseline];
+    if (baseline.valid && baseline.metric == result.metric) {
+        // divide by the baseline
+        if (!corrected.valid)
+            corrected = result;
+        corrected.value /= corrected.iterations;
+        corrected.iterations = 1;
+        corrected.metric = QTest::Percentage;
+
+        qreal subtract = 0;
+        if (zero.valid && zero.metric == baseline.metric)
+            subtract = zero.value / zero.iterations;
+        corrected.value /= baseline.value / baseline.iterations - subtract;
+    }
+
+    QTest::testLogger->addBenchmarkResult(result, corrected);
 }
 
 void QTestLog::startLogging(unsigned int randomSeed)
diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h
index a892d3d..02c0dd9 100644
--- a/src/testlib/qtestlog_p.h
+++ b/src/testlib/qtestlog_p.h
@@ -74,7 +74,7 @@ public:
     static void addXPass(const char *msg, const char *file, int line);
     static void addSkip(const char *msg, QTest::SkipMode mode,
                         const char *file, int line);
-    static void addBenchmarkResult(const QBenchmarkResult &result);
+    static void addBenchmarkResult(const QBenchmarkResult &result, const QBenchmarkResult *specialResults);
     static void addIgnoreMessage(QtMsgType type, const char *msg);
     static int unhandledIgnoreMessages();
     static void printUnhandledIgnoreMessages();
diff --git a/src/testlib/qtestlogger.cpp b/src/testlib/qtestlogger.cpp
index 86826f8..3962b09 100644
--- a/src/testlib/qtestlogger.cpp
+++ b/src/testlib/qtestlogger.cpp
@@ -268,7 +268,7 @@ void QTestLogger::addIncident(IncidentTypes type, const char *description,
     }
 }
 
-void QTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
+void QTestLogger::addBenchmarkResult(const QBenchmarkResult &result, const QBenchmarkResult &)
 {
     QTestElement *benchmarkElement = new QTestElement(QTest::LET_Benchmark);
 //    printf("element %i", benchmarkElement->elementType());
diff --git a/src/testlib/qtestlogger_p.h b/src/testlib/qtestlogger_p.h
index d8867de..3dc8640 100644
--- a/src/testlib/qtestlogger_p.h
+++ b/src/testlib/qtestlogger_p.h
@@ -82,7 +82,7 @@ class QTestLogger : public QAbstractTestLogger
 
         void addIncident(IncidentTypes type, const char *description,
                      const char *file = 0, int line = 0);
-        void addBenchmarkResult(const QBenchmarkResult &result);
+        void addBenchmarkResult(const QBenchmarkResult &result, const QBenchmarkResult &correctedResult);
         void addTag(QTestElement* element);
 
         void addMessage(MessageTypes type, const char *message,
diff --git a/src/testlib/qtesttable.cpp b/src/testlib/qtesttable.cpp
index df10462..7c5acce 100644
--- a/src/testlib/qtesttable.cpp
+++ b/src/testlib/qtesttable.cpp
@@ -190,9 +190,9 @@ bool QTestTable::isEmpty() const
     return !d->list;
 }
 
-QTestData *QTestTable::newData(const char *tag)
+QTestData *QTestTable::newData(const char *tag, QTest::BenchmarkDataMode benchMode)
 {
-    QTestData *dt = new QTestData(tag, this);
+    QTestData *dt = new QTestData(tag, benchMode, this);
     d->append(dt);
     return dt;
 }
diff --git a/src/testlib/qtesttable_p.h b/src/testlib/qtesttable_p.h
index f835506..b85e9aa 100644
--- a/src/testlib/qtesttable_p.h
+++ b/src/testlib/qtesttable_p.h
@@ -67,7 +67,7 @@ public:
     ~QTestTable();
 
     void addColumn(int elementType, const char *elementName);
-    QTestData *newData(const char *tag);
+    QTestData *newData(const char *tag, QTest::BenchmarkDataMode benchMode);
 
     int elementCount() const;
     int dataCount() const;
diff --git a/src/testlib/qxmltestlogger.cpp b/src/testlib/qxmltestlogger.cpp
index 827b0ad..3edd687 100644
--- a/src/testlib/qxmltestlogger.cpp
+++ b/src/testlib/qxmltestlogger.cpp
@@ -246,7 +246,7 @@ void QXmlTestLogger::addIncident(IncidentTypes type, const char *description,
     outputString(buf.constData());
 }
 
-void QXmlTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
+void QXmlTestLogger::addBenchmarkResult(const QBenchmarkResult &result, const QBenchmarkResult &)
 {
     QTestCharBuffer buf;
     QTestCharBuffer quotedMetric;
diff --git a/src/testlib/qxmltestlogger_p.h b/src/testlib/qxmltestlogger_p.h
index f815103..adfe98b 100644
--- a/src/testlib/qxmltestlogger_p.h
+++ b/src/testlib/qxmltestlogger_p.h
@@ -74,7 +74,7 @@ public:
 
     void addIncident(IncidentTypes type, const char *description,
                      const char *file = 0, int line = 0);
-    void addBenchmarkResult(const QBenchmarkResult &result);
+    void addBenchmarkResult(const QBenchmarkResult &result, const QBenchmarkResult &corrected);
 
     void addMessage(MessageTypes type, const char *message,
                     const char *file = 0, int line = 0);