summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--tests/arthur/baselineserver/src/baselineserver.cpp30
-rw-r--r--tests/arthur/baselineserver/src/htmlpage.cpp15
-rw-r--r--tests/arthur/common/baselineprotocol.cpp28
-rw-r--r--tests/arthur/common/baselineprotocol.h31
-rw-r--r--tests/auto/lancelot/tst_lancelot.cpp65
5 files changed, 81 insertions, 88 deletions
diff --git a/tests/arthur/baselineserver/src/baselineserver.cpp b/tests/arthur/baselineserver/src/baselineserver.cpp
index 53e40b6..90eb594 100644
--- a/tests/arthur/baselineserver/src/baselineserver.cpp
+++ b/tests/arthur/baselineserver/src/baselineserver.cpp
@@ -208,8 +208,8 @@ void BaselineHandler::provideBaselineChecksums(const QByteArray &itemListBlock)
ImageItemList itemList;
QDataStream ds(itemListBlock);
ds >> itemList;
- qDebug() << runId << logtime() << "Received request for checksums for" << itemList.count() << "items, engine"
- << itemList.at(0).engineAsString() << "pixel format" << itemList.at(0).formatAsString();
+ qDebug() << runId << logtime() << "Received request for checksums for" << itemList.count()
+ << "items in test function" << itemList.at(0).testFunction;
for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) {
i->imageChecksums.clear();
@@ -232,10 +232,10 @@ void BaselineHandler::provideBaselineChecksums(const QByteArray &itemListBlock)
if (file.open(QIODevice::ReadOnly)) {
QTextStream in(&file);
do {
- QString scriptName = in.readLine();
- if (!scriptName.isNull()) {
+ QString itemName = in.readLine();
+ if (!itemName.isNull()) {
for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) {
- if (i->scriptName == scriptName)
+ if (i->itemName == itemName)
i->status = ImageItem::IgnoreItem;
}
}
@@ -258,7 +258,7 @@ void BaselineHandler::storeImage(const QByteArray &itemBlock, bool isBaseline)
ds >> item;
QString prefix = pathForItem(item, isBaseline);
- qDebug() << runId << logtime() << "Received" << (isBaseline ? "baseline" : "mismatched") << "image for:" << item.scriptName << "Storing in" << prefix;
+ qDebug() << runId << logtime() << "Received" << (isBaseline ? "baseline" : "mismatched") << "image for:" << item.itemName << "Storing in" << prefix;
QString dir = prefix.section(QLC('/'), 0, -2);
QDir cwd;
@@ -333,17 +333,17 @@ QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, boo
if (mapped.isEmpty())
mapPlatformInfo();
- QString itemName = item.scriptName;
- if (itemName.contains(QLC('.')))
- itemName.replace(itemName.lastIndexOf(QLC('.')), 1, QLC('_'));
+ QString itemName = item.itemName;
+ itemName.replace(QLC('.'), QLC('_'));
itemName.append(QLC('_'));
- itemName.append(QString::number(item.scriptChecksum, 16).rightJustified(4, QLC('0')));
+ itemName.append(QString::number(item.itemChecksum, 16).rightJustified(4, QLC('0')));
QStringList path;
if (absolute)
path += BaselineServer::storagePath();
+ path += mapped.value(PI_TestCase);
path += QLS(isBaseline ? "baselines" : "mismatches");
- path += item.engineAsString() + QLC('_') + item.formatAsString();
+ path += item.testFunction;
path += mapped.value(PI_QtVersion);
path += mapped.value(PI_QMakeSpec);
path += mapped.value(PI_HostName);
@@ -397,6 +397,7 @@ QString BaselineHandler::updateSingleBaseline(const QString &oldBaseline, const
return res;
}
+
QString BaselineHandler::blacklistTest(const QString &context, const QString &itemId, bool removeFromBlacklist)
{
QFile file(BaselineServer::storagePath() + QLC('/') + context + QLS("/BLACKLIST"));
@@ -439,11 +440,10 @@ void BaselineHandler::testPathMapping()
<< QLS("localhost");
ImageItem item;
- item.scriptName = QLS("arcs.qps");
- item.engine = ImageItem::Raster;
- item.renderFormat = QImage::Format_ARGB32_Premultiplied;
+ item.testFunction = QLS("testPathMapping");
+ item.itemName = QLS("arcs.qps");
item.imageChecksums << 0x0123456789abcdefULL;
- item.scriptChecksum = 0x0123;
+ item.itemChecksum = 0x0123;
plat.insert(PI_QtVersion, QLS("4.8.0"));
plat.insert(PI_BuildKey, QLS("(nobuildkey)"));
diff --git a/tests/arthur/baselineserver/src/htmlpage.cpp b/tests/arthur/baselineserver/src/htmlpage.cpp
index 11c2eac..2b16d25 100644
--- a/tests/arthur/baselineserver/src/htmlpage.cpp
+++ b/tests/arthur/baselineserver/src/htmlpage.cpp
@@ -69,7 +69,7 @@ void HTMLPage::start(const QString &storagepath, const QString &runId, const Pla
ctx = context;
root = storagepath + QLC('/');
imageItems = itemList;
- reportDir = pinfo.value(PI_PulseGitBranch).isEmpty() ? QLS("reports/adhoc/") : QLS("reports/pulse/");
+ reportDir = pinfo.value(PI_TestCase) + QLC('/') + (pinfo.value(PI_PulseGitBranch).isEmpty() ? QLS("reports/adhoc/") : QLS("reports/pulse/"));
QString dir = root + reportDir;
QDir cwd;
if (!cwd.exists(dir))
@@ -79,8 +79,7 @@ void HTMLPage::start(const QString &storagepath, const QString &runId, const Pla
void HTMLPage::writeHeader(const ImageItem &item)
{
- path = reportDir + id + QLC('_') + item.engineAsString()
- + QLC('_') + item.formatAsString() + QLS(".html");
+ path = reportDir + id + QLC('_') + item.testFunction + QLS(".html");
QString pageUrl = BaselineServer::baseUrl() + path;
@@ -126,7 +125,7 @@ void HTMLPage::addItem(const QString &baseline, const QString &rendered, const I
QString pageUrl = BaselineServer::baseUrl() + path;
out << "<tr>\n";
- out << "<td>" << item.scriptName << "</td>\n";
+ out << "<td>" << item.itemName << "</td>\n";
QStringList images = QStringList() << baseline << rendered << compared;
foreach(const QString& img, images)
out << "<td><a href=\"/" << img << "\"><img src=\"/" << generateThumbnail(img) << "\" width=240 height=240></a></td>\n";
@@ -135,7 +134,7 @@ void HTMLPage::addItem(const QString &baseline, const QString &rendered, const I
<< "<p><a href=\"/cgi-bin/server.cgi?cmd=updateSingleBaseline&oldBaseline=" << baseline
<< "&newBaseline=" << rendered << "&url=" << pageUrl << "\">Replace baseline with rendered</a></p>\n"
<< "<p><a href=\"/cgi-bin/server.cgi?cmd=blacklist&context=" << ctx
- << "&itemId=" << item.scriptName << "&url=" << pageUrl << "\">Blacklist this item</a></p>\n"
+ << "&itemId=" << item.itemName << "&url=" << pageUrl << "\">Blacklist this item</a></p>\n"
<< "<p><a href=\"/cgi-bin/server.cgi?cmd=view&baseline=" << baseline << "&rendered=" << rendered
<< "&compared=" << compared << "&url=" << pageUrl << "\">View</a></p>\n"
<< "</td>\n";
@@ -144,7 +143,7 @@ void HTMLPage::addItem(const QString &baseline, const QString &rendered, const I
QMutableVectorIterator<ImageItem> it(imageItems);
while (it.hasNext()) {
it.next();
- if (it.value().scriptName == item.scriptName) {
+ if (it.value().itemName == item.itemName) {
it.remove();
break;
}
@@ -158,11 +157,11 @@ void HTMLPage::end()
// Add the names of the scripts that passed the test, or were blacklisted
QString pageUrl = BaselineServer::baseUrl() + path;
for (int i=0; i<imageItems.count(); ++i) {
- out << "<tr><td>" << imageItems.at(i).scriptName << "</td><td>N/A</td><td>N/A</td><td>N/A</td><td>";
+ out << "<tr><td>" << imageItems.at(i).itemName << "</td><td>N/A</td><td>N/A</td><td>N/A</td><td>";
if (imageItems.at(i).status == ImageItem::IgnoreItem) {
out << "<span style=\"background-color:yellow\">Blacklisted</span> "
<< "<a href=\"/cgi-bin/server.cgi?cmd=whitelist&context=" << ctx
- << "&itemId=" << imageItems.at(i).scriptName << "&url=" << pageUrl
+ << "&itemId=" << imageItems.at(i).itemName << "&url=" << pageUrl
<< "\">Whitelist item</a>";
} else {
out << "<span style=\"color:green\">Test passed</span>";
diff --git a/tests/arthur/common/baselineprotocol.cpp b/tests/arthur/common/baselineprotocol.cpp
index 5ed58b4..c7e9e72 100644
--- a/tests/arthur/common/baselineprotocol.cpp
+++ b/tests/arthur/common/baselineprotocol.cpp
@@ -128,11 +128,10 @@ PlatformInfo::PlatformInfo(bool useLocal)
ImageItem &ImageItem::operator=(const ImageItem &other)
{
- scriptName = other.scriptName;
- scriptChecksum = other.scriptChecksum;
+ testFunction = other.testFunction;
+ itemName = other.itemName;
+ itemChecksum = other.itemChecksum;
status = other.status;
- renderFormat = other.renderFormat;
- engine = other.engine;
image = other.image;
imageChecksums = other.imageChecksums;
return *this;
@@ -165,6 +164,7 @@ quint64 ImageItem::computeChecksum(const QImage &image)
return (quint64(h1) << 32) | h2;
}
+#if 0
QString ImageItem::engineAsString() const
{
switch (engine) {
@@ -205,6 +205,7 @@ QString ImageItem::formatAsString() const
return QLS("UnknownFormat");
return QLS(formatNames[renderFormat]);
}
+#endif
void ImageItem::writeImageToStream(QDataStream &out) const
{
@@ -249,20 +250,16 @@ void ImageItem::readImageFromStream(QDataStream &in)
QDataStream & operator<< (QDataStream &stream, const ImageItem &ii)
{
- stream << ii.scriptName << ii.scriptChecksum << quint8(ii.status) << quint8(ii.renderFormat)
- << quint8(ii.engine) << ii.imageChecksums;
+ stream << ii.testFunction << ii.itemName << ii.itemChecksum << quint8(ii.status) << ii.imageChecksums;
ii.writeImageToStream(stream);
return stream;
}
QDataStream & operator>> (QDataStream &stream, ImageItem &ii)
{
- quint8 encFormat, encStatus, encEngine;
- stream >> ii.scriptName >> ii.scriptChecksum >> encStatus >> encFormat
- >> encEngine >> ii.imageChecksums;
- ii.renderFormat = QImage::Format(encFormat);
+ quint8 encStatus;
+ stream >> ii.testFunction >> ii.itemName >> ii.itemChecksum >> encStatus >> ii.imageChecksums;
ii.status = ImageItem::ItemStatus(encStatus);
- ii.engine = ImageItem::GraphicsEngine(encEngine);
ii.readImageFromStream(stream);
return stream;
}
@@ -275,7 +272,7 @@ BaselineProtocol::~BaselineProtocol()
}
-bool BaselineProtocol::connect(bool *dryrun)
+bool BaselineProtocol::connect(const QString &testCase, bool *dryrun)
{
errMsg.clear();
QByteArray serverName(qgetenv("QT_LANCELOT_SERVER"));
@@ -292,6 +289,7 @@ bool BaselineProtocol::connect(bool *dryrun)
}
PlatformInfo pi(true);
+ pi.insert(PI_TestCase, testCase);
QByteArray block;
QDataStream ds(&block, QIODevice::ReadWrite);
ds << pi;
@@ -342,11 +340,15 @@ bool BaselineProtocol::acceptConnection(PlatformInfo *pi)
}
-bool BaselineProtocol::requestBaselineChecksums(ImageItemList *itemList)
+bool BaselineProtocol::requestBaselineChecksums(const QString &testFunction, ImageItemList *itemList)
{
errMsg.clear();
if (!itemList)
return false;
+
+ for(ImageItemList::iterator it = itemList->begin(); it != itemList->end(); it++)
+ it->testFunction = testFunction;
+
QByteArray block;
QDataStream ds(&block, QIODevice::ReadWrite);
ds << *itemList;
diff --git a/tests/arthur/common/baselineprotocol.h b/tests/arthur/common/baselineprotocol.h
index baffb4a..1f4d593 100644
--- a/tests/arthur/common/baselineprotocol.h
+++ b/tests/arthur/common/baselineprotocol.h
@@ -52,6 +52,7 @@
#define FileFormat "png"
+const QString PI_TestCase(QLS("TestCase"));
const QString PI_HostName(QLS("HostName"));
const QString PI_HostAddress(QLS("HostAddress"));
const QString PI_OSName(QLS("OSName"));
@@ -73,19 +74,15 @@ struct ImageItem
{
public:
ImageItem()
- : status(Ok), renderFormat(QImage::Format_Invalid), engine(Raster), scriptChecksum(0)
+ : status(Ok), itemChecksum(0)
{}
ImageItem(const ImageItem &other)
{ *this = other; }
~ImageItem()
{}
ImageItem &operator=(const ImageItem &other);
- static quint64 computeChecksum(const QImage& image);
- QString engineAsString() const;
- QString formatAsString() const;
- void writeImageToStream(QDataStream &stream) const;
- void readImageFromStream(QDataStream &stream);
+ static quint64 computeChecksum(const QImage& image);
enum ItemStatus {
Ok = 0,
@@ -93,19 +90,15 @@ public:
IgnoreItem = 2
};
- enum GraphicsEngine {
- Raster = 0,
- OpenGL = 1
- };
-
- QString scriptName;
+ QString testFunction;
+ QString itemName;
ItemStatus status;
- QImage::Format renderFormat;
- GraphicsEngine engine;
QImage image;
QList<quint64> imageChecksums;
- // tbd: add diffscore
- quint16 scriptChecksum;
+ quint16 itemChecksum;
+
+ void writeImageToStream(QDataStream &stream) const;
+ void readImageFromStream(QDataStream &stream);
};
QDataStream & operator<< (QDataStream &stream, const ImageItem &ii);
QDataStream & operator>> (QDataStream &stream, ImageItem& ii);
@@ -124,7 +117,7 @@ public:
// Important constants here
// ****************************************************
enum Constant {
- ProtocolVersion = 3,
+ ProtocolVersion = 4,
ServerPort = 54129,
Timeout = 5000
};
@@ -143,8 +136,8 @@ public:
};
// For client:
- bool connect(bool *dryrun = 0);
- bool requestBaselineChecksums(ImageItemList *itemList);
+ bool connect(const QString &testCase, bool *dryrun = 0);
+ bool requestBaselineChecksums(const QString &testFunction, ImageItemList *itemList);
bool submitNewBaseline(const ImageItem &item, QByteArray *serverMsg);
bool submitMismatch(const ImageItem &item, QByteArray *serverMsg);
diff --git a/tests/auto/lancelot/tst_lancelot.cpp b/tests/auto/lancelot/tst_lancelot.cpp
index 7c6fe66..d1b093f 100644
--- a/tests/auto/lancelot/tst_lancelot.cpp
+++ b/tests/auto/lancelot/tst_lancelot.cpp
@@ -66,10 +66,15 @@ public:
static bool simfail;
private:
- ImageItem render(const ImageItem &item);
+ enum GraphicsEngine {
+ Raster = 0,
+ OpenGL = 1
+ };
+
+ bool setupTestSuite(const QStringList& blacklist);
+ void runTestSuite(GraphicsEngine engine, QImage::Format format);
+ ImageItem render(const ImageItem &item, GraphicsEngine engine, QImage::Format format);
void paint(QPaintDevice *device, const QStringList &script, const QString &filePath);
- void runTestSuite();
- bool setupTestSuite(ImageItem::GraphicsEngine engine, QImage::Format format, const QStringList& blacklist);
BaselineProtocol proto;
ImageItemList baseList;
@@ -108,7 +113,7 @@ void tst_Lancelot::initTestCase()
#if defined(Q_OS_SOMEPLATFORM)
QSKIP("This test is not supported on this platform.", SkipAll);
#endif
- if (!proto.connect(&dryRunMode))
+ if (!proto.connect(QLatin1String("tst_Lancelot"), &dryRunMode))
QSKIP(qPrintable(proto.errorMessage()), SkipAll);
QDir qpsDir(scriptsDir);
@@ -125,8 +130,8 @@ void tst_Lancelot::initTestCase()
file.open(QFile::ReadOnly);
QByteArray cont = file.readAll();
scripts.insert(fileName, QString::fromLatin1(cont).split(QLatin1Char('\n'), QString::SkipEmptyParts));
- it->scriptName = fileName;
- it->scriptChecksum = qChecksum(cont.constData(), cont.size());
+ it->itemName = fileName;
+ it->itemChecksum = qChecksum(cont.constData(), cont.size());
it++;
}
}
@@ -135,42 +140,42 @@ void tst_Lancelot::initTestCase()
void tst_Lancelot::testRasterARGB32PM_data()
{
QStringList localBlacklist;
- if (!setupTestSuite(ImageItem::Raster, QImage::Format_ARGB32_Premultiplied, localBlacklist))
+ if (!setupTestSuite(localBlacklist))
QSKIP("Communication with baseline image server failed.", SkipAll);
}
void tst_Lancelot::testRasterARGB32PM()
{
- runTestSuite();
+ runTestSuite(Raster, QImage::Format_ARGB32_Premultiplied);
}
void tst_Lancelot::testRasterRGB32_data()
{
QStringList localBlacklist;
- if (!setupTestSuite(ImageItem::Raster, QImage::Format_RGB32, localBlacklist))
+ if (!setupTestSuite(localBlacklist))
QSKIP("Communication with baseline image server failed.", SkipAll);
}
void tst_Lancelot::testRasterRGB32()
{
- runTestSuite();
+ runTestSuite(Raster, QImage::Format_RGB32);
}
void tst_Lancelot::testRasterRGB16_data()
{
QStringList localBlacklist;
- if (!setupTestSuite(ImageItem::Raster, QImage::Format_RGB16, localBlacklist))
+ if (!setupTestSuite(localBlacklist))
QSKIP("Communication with baseline image server failed.", SkipAll);
}
void tst_Lancelot::testRasterRGB16()
{
- runTestSuite();
+ runTestSuite(Raster, QImage::Format_RGB16);
}
@@ -178,7 +183,7 @@ void tst_Lancelot::testRasterRGB16()
void tst_Lancelot::testOpenGL_data()
{
QStringList localBlacklist = QStringList() << QLatin1String("rasterops.qps");
- if (!setupTestSuite(ImageItem::OpenGL, QImage::Format_RGB32, localBlacklist))
+ if (!setupTestSuite(localBlacklist))
QSKIP("Communication with baseline image server failed.", SkipAll);
}
@@ -197,45 +202,39 @@ void tst_Lancelot::testOpenGL()
ok = true;
}
if (ok)
- runTestSuite();
+ runTestSuite(OpenGL, QImage::Format_RGB32);
else
QSKIP("System under test does not meet preconditions for GL testing. Skipping.", SkipAll);
}
#endif
-bool tst_Lancelot::setupTestSuite(ImageItem::GraphicsEngine engine, QImage::Format format, const QStringList& blacklist)
+bool tst_Lancelot::setupTestSuite(const QStringList& blacklist)
{
QTest::addColumn<ImageItem>("baseline");
ImageItemList itemList(baseList);
-
- for(ImageItemList::iterator it = itemList.begin(); it != itemList.end(); it++) {
- it->engine = engine;
- it->renderFormat = format;
- }
-
- if (!proto.requestBaselineChecksums(&itemList)) {
+ if (!proto.requestBaselineChecksums(QTest::currentTestFunction(), &itemList)) {
QWARN(qPrintable(proto.errorMessage()));
return false;
}
foreach(const ImageItem& item, itemList) {
- if (!blacklist.contains(item.scriptName))
- QTest::newRow(item.scriptName.toLatin1()) << item;
+ if (!blacklist.contains(item.itemName))
+ QTest::newRow(item.itemName.toLatin1()) << item;
}
return true;
}
-void tst_Lancelot::runTestSuite()
+void tst_Lancelot::runTestSuite(GraphicsEngine engine, QImage::Format format)
{
QFETCH(ImageItem, baseline);
if (baseline.status == ImageItem::IgnoreItem)
QSKIP("Blacklisted by baseline server.", SkipSingle);
- ImageItem rendered = render(baseline);
+ ImageItem rendered = render(baseline, engine, format);
if (rendered.image.isNull()) { // Assume an error in the test environment, not Qt
QWARN("Error: Failed to render image.");
QSKIP("Aborted due to errors.", SkipSingle);
@@ -258,21 +257,21 @@ void tst_Lancelot::runTestSuite()
}
-ImageItem tst_Lancelot::render(const ImageItem &item)
+ImageItem tst_Lancelot::render(const ImageItem &item, GraphicsEngine engine, QImage::Format format)
{
ImageItem res = item;
res.imageChecksums.clear();
res.image = QImage();
- QString filePath = scriptsDir + item.scriptName;
- QStringList script = scripts.value(item.scriptName);
+ QString filePath = scriptsDir + item.itemName;
+ QStringList script = scripts.value(item.itemName);
- if (item.engine == ImageItem::Raster) {
- QImage img(800, 800, item.renderFormat);
+ if (engine == Raster) {
+ QImage img(800, 800, format);
paint(&img, script, QFileInfo(filePath).absoluteFilePath()); // eh yuck (filePath stuff)
res.image = img;
res.imageChecksums.append(ImageItem::computeChecksum(img));
#ifndef QT_NO_OPENGL
- } else if (item.engine == ImageItem::OpenGL) {
+ } else if (engine == OpenGL) {
QGLWidget glWidget;
if (glWidget.isValid()) {
glWidget.makeCurrent();
@@ -281,7 +280,7 @@ ImageItem tst_Lancelot::render(const ImageItem &item)
fboFormat.setAttachment(QGLFramebufferObject::CombinedDepthStencil);
QGLFramebufferObject fbo(800, 800, fboFormat);
paint(&fbo, script, QFileInfo(filePath).absoluteFilePath()); // eh yuck (filePath stuff)
- res.image = fbo.toImage().convertToFormat(item.renderFormat);
+ res.image = fbo.toImage().convertToFormat(format);
res.imageChecksums.append(ImageItem::computeChecksum(res.image));
}
#endif
generated by cgit v0.12 at 2024-11-19 11:45:56 (GMT)