summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--tests/arthur/baselineserver/src/baselineserver.cpp78
-rw-r--r--tests/arthur/baselineserver/src/baselineserver.h5
-rw-r--r--tests/arthur/common/baselineprotocol.cpp78
-rw-r--r--tests/arthur/common/baselineprotocol.h63
-rw-r--r--tests/auto/lancelot/tst_lancelot.cpp131
5 files changed, 265 insertions, 90 deletions
diff --git a/tests/arthur/baselineserver/src/baselineserver.cpp b/tests/arthur/baselineserver/src/baselineserver.cpp
index 9f1665f..3087357 100644
--- a/tests/arthur/baselineserver/src/baselineserver.cpp
+++ b/tests/arthur/baselineserver/src/baselineserver.cpp
@@ -5,6 +5,7 @@
#include <QCoreApplication>
#include <QFileInfo>
#include <QHostInfo>
+#include <QTextStream>
QString BaselineServer::storage;
@@ -106,6 +107,9 @@ void BaselineHandler::receiveRequest()
}
switch(cmd) {
+ case BaselineProtocol::RequestBaselineChecksums:
+ provideBaselineChecksums(block);
+ break;
case BaselineProtocol::RequestBaseline:
provideBaseline(block);
break;
@@ -122,40 +126,73 @@ void BaselineHandler::receiveRequest()
}
+void BaselineHandler::provideBaselineChecksums(const QByteArray &itemListBlock)
+{
+ ImageItemList itemList;
+ QDataStream ds(itemListBlock);
+ ds >> itemList;
+ qDebug() << runId << logtime() << "Received request for checksums for" << itemList.count() << "items";
+
+ for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) {
+ i->imageChecksum = 0;
+ QString prefix = pathForItem(*i, true);
+ QFile file(prefix + QLatin1String("metadata"));
+ if (file.open(QIODevice::ReadOnly)) {
+ QTextStream ts(&file);
+ ts >> i->imageChecksum;
+ file.close();
+ i->status = ImageItem::Ok;
+ }
+ if (!i->imageChecksum)
+ i->status = ImageItem::BaselineNotFound;
+ }
+
+ QByteArray block;
+ QDataStream ods(&block, QIODevice::WriteOnly);
+ ods << itemList;
+ proto.sendBlock(BaselineProtocol::Ack, block);
+}
+
+
void BaselineHandler::provideBaseline(const QByteArray &caseId)
{
qDebug() << runId << logtime() << "Received request for baseline" << caseId;
+ //### rewrite to use ImageItem
+ /*
QFile file(pathForCaseId(caseId));
if (!file.open(QIODevice::ReadOnly)) {
qDebug() << runId << logtime() << "baseline not found.";
proto.sendBlock(BaselineProtocol::BaselineNotPresent, caseId);
return;
}
+
proto.sendBlock(BaselineProtocol::AcceptBaseline, file.readAll());
+ */
}
-void BaselineHandler::storeImage(const QByteArray &imageBlock, bool isBaseline)
+void BaselineHandler::storeImage(const QByteArray &itemBlock, bool isBaseline)
{
- QBuffer buf;
- buf.setData(imageBlock);
- buf.open(QIODevice::ReadOnly);
- QDataStream ds(&buf);
- QByteArray caseId;
- ds >> caseId;
- //# For futuresafeness, should make caseId FS-safe - but revertable...
- QFile file(pathForCaseId(caseId, isBaseline));
- qDebug() << runId << logtime() << "Received" << (isBaseline ? "baseline" : "mismatched") << "image for:" << caseId << "Storing in" << file.fileName();
- QString path = file.fileName().section(QDir::separator(), 0, -2);
+ QDataStream ds(itemBlock);
+ ImageItem item;
+ ds >> item;
+
+ QString prefix = pathForItem(item, isBaseline);
+ qDebug() << runId << logtime() << "Received" << (isBaseline ? "baseline" : "mismatched") << "image for:" << item.scriptName << "Storing in" << prefix;
+
+ QString dir = prefix.section(QDir::separator(), 0, -2);
QDir cwd;
- if (!cwd.exists(path))
- cwd.mkpath(path);
- if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
- qWarning() << runId << logtime() << "Failed to store" << file.fileName();
- return;
- }
- file.write(imageBlock.constData()+buf.pos(), imageBlock.size()-buf.pos());
+ if (!cwd.exists(dir))
+ cwd.mkpath(dir);
+ item.image.save(prefix + QLatin1String(FileFormat), FileFormat);
+
+ //# Could use QSettings or XML or even DB, could use common file for whole dir or even whole storage - but for now, keep it simple
+ QFile file(prefix + QLatin1String("metadata"));
+ file.open(QIODevice::WriteOnly | QIODevice::Truncate);
+ QTextStream ts(&file);
+ ts << hex << showbase << item.imageChecksum << reset << endl;
+ file.close();
QByteArray msg(isBaseline ? "Baseline" : "Mismatching" );
msg += " image stored in "
@@ -174,7 +211,7 @@ void BaselineHandler::receiveDisconnect()
}
-QString BaselineHandler::pathForCaseId(const QByteArray &caseId, bool isBaseline)
+QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline)
{
QString storePath = BaselineServer::storagePath();
storePath += plat.buildKey.section(QLatin1Char(' '), 1, 1) + QLatin1String("_Qt-")
@@ -183,7 +220,8 @@ QString BaselineHandler::pathForCaseId(const QByteArray &caseId, bool isBaseline
storePath += QLatin1String("baselines") + QDir::separator();
else
storePath += runId + QDir::separator();
- return storePath + caseId + QLatin1Char('.') + QLatin1String(FileFormat);
+ //#? QString itemName = item.scriptName.replace(item.scriptName.lastIndexOf('.'), '_');
+ return storePath + item.scriptName + QLatin1Char('.');
}
diff --git a/tests/arthur/baselineserver/src/baselineserver.h b/tests/arthur/baselineserver/src/baselineserver.h
index 9d32e5d..4056ce0 100644
--- a/tests/arthur/baselineserver/src/baselineserver.h
+++ b/tests/arthur/baselineserver/src/baselineserver.h
@@ -11,7 +11,7 @@
#include <QDateTime>
// #seconds between update checks
-#define HEARTBEAT 10
+#define HEARTBEAT 5
class BaselineServer : public QTcpServer
{
@@ -61,9 +61,10 @@ private slots:
void receiveDisconnect();
private:
+ void provideBaselineChecksums(const QByteArray &itemListBlock);
void provideBaseline(const QByteArray &caseId);
void storeImage(const QByteArray &imageBlock, bool isBaseline);
- QString pathForCaseId(const QByteArray &caseId, bool isBaseline = true);
+ QString pathForItem(const ImageItem &item, bool isBaseline = true);
QString logtime();
BaselineProtocol proto;
diff --git a/tests/arthur/common/baselineprotocol.cpp b/tests/arthur/common/baselineprotocol.cpp
index 83b7adc..e5545a9 100644
--- a/tests/arthur/common/baselineprotocol.cpp
+++ b/tests/arthur/common/baselineprotocol.cpp
@@ -26,6 +26,42 @@ QDataStream & operator>> (QDataStream& stream, PlatformInfo& pinfo)
return stream;
}
+ImageItem &ImageItem::operator=(const ImageItem &other)
+{
+ scriptName = other.scriptName;
+ scriptChecksum = other.scriptChecksum;
+ status = other.status;
+ renderFormat = other.renderFormat;
+ engine = other.engine;
+ image = other.image;
+ imageChecksum = other.imageChecksum;
+ return *this;
+}
+
+quint64 ImageItem::computeChecksum(const QImage &image)
+{
+ //### Just fake it for now
+ return qChecksum((const char *)image.constScanLine(50), image.bytesPerLine());
+}
+
+QDataStream & operator<< (QDataStream &stream, const ImageItem &ii)
+{
+ stream << ii.scriptName << ii.scriptChecksum << quint8(ii.status) << quint8(ii.renderFormat)
+ << quint8(ii.engine) << ii.image << ii.imageChecksum;
+ return stream;
+}
+
+QDataStream & operator>> (QDataStream &stream, ImageItem &ii)
+{
+ quint8 encFormat, encStatus, encEngine;
+ stream >> ii.scriptName >> ii.scriptChecksum >> encStatus >> encFormat
+ >> encEngine >> ii.image >> ii.imageChecksum;
+ ii.renderFormat = QImage::Format(encFormat);
+ ii.status = ImageItem::ItemStatus(encStatus);
+ ii.engine = ImageItem::GraphicsEngine(encEngine);
+ return stream;
+}
+
BaselineProtocol::~BaselineProtocol()
{
socket.close();
@@ -86,8 +122,28 @@ bool BaselineProtocol::acceptConnection(PlatformInfo *pi)
}
+bool BaselineProtocol::requestBaselineChecksums(ImageItemList *itemList)
+{
+ errMsg.clear();
+ if (!itemList)
+ return false;
+ QByteArray block;
+ QDataStream ds(&block, QIODevice::ReadWrite);
+ ds << *itemList;
+ if (!sendBlock(RequestBaselineChecksums, block))
+ return false;
+ Command cmd;
+ if (!receiveBlock(&cmd, &block))
+ return false;
+ ds.device()->seek(0);
+ ds >> *itemList;
+ return true;
+}
+
+
bool BaselineProtocol::requestBaseline(const QString &caseId, Command *response, QImage *baseline)
{
+ //### TBD: rewrite to use ImageItem
errMsg.clear();
if (!sendBlock(RequestBaseline, caseId.toLatin1()))
return false;
@@ -115,29 +171,31 @@ bool BaselineProtocol::requestBaseline(const QString &caseId, Command *response,
}
-bool BaselineProtocol::submitNewBaseline(const QString &caseId, const QImage &baseline)
+bool BaselineProtocol::submitNewBaseline(const ImageItem &item)
+{
+ return sendItem(AcceptNewBaseline, item);
+}
+
+
+bool BaselineProtocol::sendItem(Command cmd, const ImageItem &item)
{
errMsg.clear();
QBuffer buf;
buf.open(QIODevice::WriteOnly);
QDataStream ds(&buf);
- ds << caseId.toLatin1();
- if (!baseline.save(&buf, FileFormat)) {
- errMsg = QLatin1String("Failed to convert new baseline image to ") + QLatin1String(FileFormat);
- return false;
- }
- if (!sendBlock(AcceptNewBaseline, buf.data())) {
- errMsg.prepend(QLatin1String("Failed to submit new baseline to server. "));
+ ds << item;
+ if (!sendBlock(cmd, buf.data())) {
+ errMsg.prepend(QLatin1String("Failed to submit image to server. "));
return false;
}
- Command cmd;
- receiveBlock(&cmd, 0); // Just wait for the pong; ignore reply contents
+ receiveBlock(&cmd, 0); // For now, Just wait for the pong; ignore reply contents
return true;
}
bool BaselineProtocol::submitMismatch(const QString &caseId, const QImage &mismatch, QByteArray *failMsg)
{
+ //### TBD: rewrite to use ImageItem
errMsg.clear();
QBuffer buf;
buf.open(QIODevice::WriteOnly);
diff --git a/tests/arthur/common/baselineprotocol.h b/tests/arthur/common/baselineprotocol.h
index 8740413..b16a55b 100644
--- a/tests/arthur/common/baselineprotocol.h
+++ b/tests/arthur/common/baselineprotocol.h
@@ -3,19 +3,61 @@
#include <QDataStream>
#include <QTcpSocket>
+#include <QImage>
+#include <QVector>
#define FileFormat "png"
struct PlatformInfo
{
-public:
PlatformInfo(bool useLocal = false);
QString buildKey;
QString qtVersion;
QString hostname;
};
+QDataStream & operator<< (QDataStream &stream, const PlatformInfo &pinfo);
+QDataStream & operator>> (QDataStream& stream, PlatformInfo& pinfo);
+struct ImageItem
+{
+public:
+ ImageItem()
+ : status(Ok), renderFormat(QImage::Format_Invalid), engine(Raster), imageChecksum(0), scriptChecksum(0)
+ {}
+ ImageItem(const ImageItem &other)
+ { *this = other; }
+ ~ImageItem()
+ {}
+ ImageItem &operator=(const ImageItem &other);
+ static quint64 computeChecksum(const QImage& image);
+
+ enum ItemStatus {
+ Ok = 0,
+ BaselineNotFound = 1,
+ IgnoreItem = 2
+ };
+
+ enum GraphicsEngine {
+ Raster = 0,
+ OpenGL = 1
+ };
+
+ QString scriptName;
+ ItemStatus status;
+ QImage::Format renderFormat;
+ GraphicsEngine engine;
+ QImage image;
+ quint64 imageChecksum;
+ // tbd: add diffscore
+ quint16 scriptChecksum;
+};
+QDataStream & operator<< (QDataStream &stream, const ImageItem &ii);
+QDataStream & operator>> (QDataStream& stream, ImageItem& ii);
+
+Q_DECLARE_METATYPE(ImageItem);
+
+typedef QVector<ImageItem> ImageItemList;
class BaselineProtocol
@@ -30,18 +72,21 @@ public:
enum Constant {
ProtocolVersion = 1,
ServerPort = 54129,
- Timeout = 100000
+ Timeout = 10000
};
enum Command {
UnknownError = 0,
// Queries
AcceptPlatformInfo = 1,
- RequestBaseline = 2,
- AcceptNewBaseline = 3,
- AcceptMismatch = 4,
+ RequestBaselineChecksums = 2,
+ RequestBaseline = 3,
+ AcceptNewBaseline = 4,
+ AcceptMismatch = 5,
// Responses
Ack = 128,
+
+ //#### remove these:
AcceptBaseline = 129,
BaselineNotPresent = 130,
IgnoreCase = 131
@@ -49,8 +94,9 @@ public:
// For client:
bool connect();
+ bool requestBaselineChecksums(ImageItemList *itemList);
bool requestBaseline(const QString &caseId, Command *response, QImage *baseline);
- bool submitNewBaseline(const QString &caseId, const QImage &baseline);
+ bool submitNewBaseline(const ImageItem &item);
bool submitMismatch(const QString &caseId, const QImage &mismatch, QByteArray *failMsg);
// For server:
@@ -59,6 +105,8 @@ public:
QString errorMessage();
private:
+ bool sendItem(Command cmd, const ImageItem &item);
+
bool sendBlock(Command cmd, const QByteArray &block);
bool receiveBlock(Command *cmd, QByteArray *block);
QString errMsg;
@@ -69,8 +117,7 @@ private:
};
-QDataStream & operator<< (QDataStream &stream, const PlatformInfo &pinfo);
-QDataStream & operator>> (QDataStream& stream, PlatformInfo& pinfo);
+
#endif // BASELINEPROTOCOL_H
diff --git a/tests/auto/lancelot/tst_lancelot.cpp b/tests/auto/lancelot/tst_lancelot.cpp
index 5bbc575..a6bb68f 100644
--- a/tests/auto/lancelot/tst_lancelot.cpp
+++ b/tests/auto/lancelot/tst_lancelot.cpp
@@ -65,19 +65,21 @@ public:
private:
QImage getBaseline(const QString &fileName, bool *created);
- QImage render(const QString &fileName);
+ QImage render(const QString &fileName, QImage::Format format);
QStringList loadScriptFile(const QString &filePath);
QString computeMismatchScore(const QImage& baseline, const QImage& rendered);
+ void runTestSuite();
QString errorMsg;
BaselineProtocol proto;
+ ImageItemList baseList;
private slots:
void initTestCase();
void cleanupTestCase() {}
- void testRendering_data();
- void testRendering();
+ void testRasterARGB32PM_data();
+ void testRasterARGB32PM();
};
tst_Lancelot::tst_Lancelot()
@@ -86,85 +88,111 @@ tst_Lancelot::tst_Lancelot()
void tst_Lancelot::initTestCase()
{
+ // Check and setup the environment. We treat failures because of test environment
+ // (e.g. script files not found) as just warnings, and not QFAILs, to avoid false negatives
+ // caused by environment or server instability
+
#if !defined(Q_OS_LINUX)
QSKIP("For the moment, this test is only supported on Linux.", SkipAll);
#endif
-
- // Check the environment here, so that we can just skip on trouble!
if (!proto.connect()) {
QWARN(qPrintable(proto.errorMessage()));
QSKIP("Communication with baseline image server failed.", SkipAll);
}
- // TBD: preload scripts, to confirm reading and generate checksums
+ QDir qpsDir(scriptsDir);
+ QStringList files = qpsDir.entryList(QStringList() << QLatin1String("*.qps"), QDir::Files | QDir::Readable);
+ if (files.isEmpty()) {
+ QWARN("No qps script files found in " + qpsDir.path().toLatin1());
+ QSKIP("Aborted due to errors.", SkipAll);
+ }
+
+ baseList.resize(files.count());
+ int i = 0;
+ foreach(const QString& file, files) {
+ baseList[i].scriptName = file;
+ // tbd: also load, split and generate checksums for scripts
+ i++;
+ }
}
-void tst_Lancelot::testRendering_data()
+
+void tst_Lancelot::testRasterARGB32PM_data()
{
- // search the data directory for qps files and make a row for each.
- // filter out those marked as blacklisted (in the file itself?)
- // Rather do the above in initTestCase. Can load & split them all also.
- // And baselines too?
+ QTest::addColumn<ImageItem>("item");
- QTest::addColumn<QString>("fileName");
+ ImageItemList itemList(baseList);
- // ### no, don't do it like this, we want opt-in, not opt-out, for stability
- QDir qpsDir(scriptsDir);
- QStringList files = qpsDir.entryList(QStringList() << QLatin1String("*.qps"), QDir::Files | QDir::Readable);
- files.removeOne(QLatin1String("porter_duff.qps"));
- files.removeOne(QLatin1String("porter_duff2.qps"));
- files.removeOne(QLatin1String("images.qps"));
- files.removeOne(QLatin1String("sizes.qps"));
+ for(int i = 0; i < itemList.size(); i++) {
+ itemList[i].engine = ImageItem::Raster;
+ itemList[i].renderFormat = QImage::Format_ARGB32_Premultiplied;
+ }
- if (files.isEmpty()) {
- QWARN("No qps files found in " + qpsDir.path().toLatin1());
- QSKIP("Aborted due to errors.", SkipAll);
+ if (!proto.requestBaselineChecksums(&itemList)) {
+ QWARN(qPrintable(proto.errorMessage()));
+ QSKIP("Communication with baseline image server failed.", SkipAll);
}
- foreach(QString fileName, files) {
- QString baseName = fileName.section('.', 0, -2);
- QTest::newRow(baseName.toLatin1()) << fileName;
+ qDebug() << "items:" << itemList.count();
+ foreach(const ImageItem& item, itemList) {
+ if (item.scriptName != QLatin1String("sizes.qps")) // Hardcoded blacklisting for this enigine/format
+ QTest::newRow(item.scriptName.toLatin1()) << item;
}
}
-// We treat failures because of test environment
-// (e.g. script files not found) as just warnings, and not
-// QFAILs, to avoid false negatives caused by environment
-// or server instability
+void tst_Lancelot::testRasterARGB32PM()
+{
+ runTestSuite();
+}
-void tst_Lancelot::testRendering()
+
+void tst_Lancelot::runTestSuite()
{
errorMsg.clear();
- QFETCH(QString, fileName);
+ QFETCH(ImageItem, item);
+
+ if (item.status == ImageItem::IgnoreItem)
+ QSKIP("Blacklisted by baseline server.", SkipSingle);
+
+ QImage rendered = render(item.scriptName, item.renderFormat);
+ if (!errorMsg.isNull() || rendered.isNull()) { // Assume an error in the test environment, not Qt
+ QWARN("Error: Failed to render image. " + errorMsg.toLatin1());
+ QSKIP("Aborted due to errors.", SkipSingle);
+ }
- bool newBaselineCreated = false;
- QImage baseline = getBaseline(fileName, &newBaselineCreated);
- if (newBaselineCreated)
+ quint64 checksum = ImageItem::computeChecksum(rendered);
+
+ if (item.status == ImageItem::BaselineNotFound) {
+ item.image = rendered;
+ item.imageChecksum = checksum;
+ proto.submitNewBaseline(item);
QSKIP("Baseline not found; new baseline created.", SkipSingle);
+ }
+
+ qDebug() << "rendered" << QString::number(checksum, 16).prepend("0x") << "baseline" << QString::number(item.imageChecksum, 16).prepend("0x");
+ if (checksum != item.imageChecksum) {
+ /* TBD
+ - Get baseline image
+ - compute diffscore
+ - submit mismatching
+
+ getBaseline(...)
if (!errorMsg.isNull() || baseline.isNull()) {
QWARN("Error: Failed to get baseline image. " + errorMsg.toLatin1());
QSKIP("Aborted due to errors.", SkipSingle);
}
- QImage rendered = render(fileName);
- if (!errorMsg.isNull() || rendered.isNull()) {
- // Indicates that it's an error in the test environment, not Qt
- QWARN("Error: Failed to render image. " + errorMsg.toLatin1());
- QSKIP("Aborted due to errors.", SkipSingle);
- }
if (rendered.format() != baseline.format())
- baseline = baseline.convertToFormat(rendered.format());
+ rendered = rendered.convertToFormat(baseline.format()); //### depending on format
- // The actual check:
- if (rendered != baseline) {
QString scoreMsg = computeMismatchScore(baseline, rendered);
QByteArray serverMsg;
proto.submitMismatch(fileName, rendered, &serverMsg);
-
- QByteArray failMsg = QByteArray("Rendered image differs from baseline. ")
- + scoreMsg.toLatin1() + '\n' + serverMsg;
+ */
+ QByteArray failMsg = QByteArray("Rendered image differs from baseline. ");
+ //+ scoreMsg.toLatin1() + '\n' + serverMsg;
QFAIL(failMsg.constData());
}
}
@@ -211,11 +239,11 @@ QString tst_Lancelot::computeMismatchScore(const QImage &baseline, const QImage
double pcd = 100.0 * ncd / (w*h); // percent of pixels that differ
double acd = ncd ? double(scd) / (3*ncd) : 0; // avg. difference
- QString res = QString(QLatin1String("Diffscore: %1% (Num:%2 Avg:%3.)")).arg(pcd, 0, 'g', 2).arg(ncd).arg(acd, 0, 'g', 2);
+ QString res = QString(QLatin1String("Diffscore: %1% (Num:%2 Avg:%3)")).arg(pcd, 0, 'g', 2).arg(ncd).arg(acd, 0, 'g', 2);
if (baseline.hasAlphaChannel()) {
double pad = 100.0 * nad / (w*h); // percent of pixels that differ
double aad = nad ? double(sad) / (3*nad) : 0; // avg. difference
- res += QString(QLatin1String(" Alpha-diffscore: %1% (Num:%2 Avg:%3.)")).arg(pad, 0, 'g', 2).arg(nad).arg(aad, 0, 'g', 2);
+ res += QString(QLatin1String(" Alpha-diffscore: %1% (Num:%2 Avg:%3)")).arg(pad, 0, 'g', 2).arg(nad).arg(aad, 0, 'g', 2);
}
return res;
}
@@ -224,6 +252,8 @@ QString tst_Lancelot::computeMismatchScore(const QImage &baseline, const QImage
QImage tst_Lancelot::getBaseline(const QString &fileName, bool *created)
{
QImage baseline;
+ //### TBD: rewrite to use ImageItem
+ /*
BaselineProtocol::Command response;
if (!proto.requestBaseline(fileName, &response, &baseline)) {
errorMsg = QLatin1String("Error downloading baseline from server. ") + proto.errorMessage();
@@ -254,18 +284,19 @@ QImage tst_Lancelot::getBaseline(const QString &fileName, bool *created)
errorMsg = QLatin1String("Unknown response from baseline server. ");
return QImage();
}
+ */
return baseline;
}
-QImage tst_Lancelot::render(const QString &fileName)
+QImage tst_Lancelot::render(const QString &fileName, QImage::Format format)
{
QString filePath = scriptsDir + fileName;
QStringList script = loadScriptFile(filePath);
if (script.isEmpty())
return QImage();
- QImage img(800, 800, QImage::Format_ARGB32);
+ QImage img(800, 800, format);
QPainter p(&img);
PaintCommands pcmd(script, 800, 800);
pcmd.setPainter(&p);
generated by cgit v0.12 at 2025-01-29 10:55:16 (GMT)