Resubmit support for subpixel antialiasing on text in the GL2 engine.

The antialiasing is currently not gamma corrected and is disabled on
OpenGL ES 2.0.

Reviewed-by: Samuel

5 files changed, 174 insertions, 23 deletions

diff --git a/src/opengl/gl2paintengineex/qglengineshadermanager.cpp b/src/opengl/gl2paintengineex/qglengineshadermanager.cpp
index b3458af..eceed06 100644
--- a/src/opengl/gl2paintengineex/qglengineshadermanager.cpp
+++ b/src/opengl/gl2paintengineex/qglengineshadermanager.cpp
@@ -134,7 +134,8 @@ QGLEngineSharedShaders::QGLEngineSharedShaders(const QGLContext* context)
         code[ShockingPinkSrcFragmentShader] = qglslShockingPinkSrcFragmentShader;
 
         code[MaskFragmentShader] = qglslMaskFragmentShader;
-        code[RgbMaskFragmentShader] = ""; //###
+        code[RgbMaskFragmentShaderPass1] = qglslRgbMaskFragmentShaderPass1;
+        code[RgbMaskFragmentShaderPass2] = qglslRgbMaskFragmentShaderPass2;
         code[RgbMaskWithGammaFragmentShader] = ""; //###
 
         code[MultiplyCompositionModeFragmentShader] = ""; //###
@@ -587,10 +588,15 @@ bool QGLEngineShaderManager::useCorrectShaderProg()
         if (maskType == PixelMask) {
             maskShaderName = QGLEngineSharedShaders::MaskFragmentShader;
             texCoords = true;
-        } else if (maskType == SubPixelMask) {
-            maskShaderName = QGLEngineSharedShaders::RgbMaskFragmentShader;
+        } else if (maskType == SubPixelMaskPass1) {
+            maskShaderName = QGLEngineSharedShaders::RgbMaskFragmentShaderPass1;
+            texCoords = true;
+        } else if (maskType == SubPixelMaskPass2) {
+            maskShaderName = QGLEngineSharedShaders::RgbMaskFragmentShaderPass2;
+            texCoords = true;
         } else if (maskType == SubPixelWithGammaMask) {
             maskShaderName = QGLEngineSharedShaders::RgbMaskWithGammaFragmentShader;
+            texCoords = true;
         } else {
             qCritical("QGLEngineShaderManager::useCorrectShaderProg() - Unknown mask type");
         }
diff --git a/src/opengl/gl2paintengineex/qglengineshadermanager_p.h b/src/opengl/gl2paintengineex/qglengineshadermanager_p.h
index 3c5a5f5..47d9a2a 100644
--- a/src/opengl/gl2paintengineex/qglengineshadermanager_p.h
+++ b/src/opengl/gl2paintengineex/qglengineshadermanager_p.h
@@ -142,7 +142,8 @@
 
     Masks are implementations of "qcolorp vec4 applyMask(qcolorp vec4 src)":
         qglslMaskFragmentShader
-        qglslRgbMaskFragmentShader
+        qglslRgbMaskFragmentShaderPass1
+        qglslRgbMaskFragmentShaderPass2
         qglslRgbMaskWithGammaFragmentShader
 
     Composition modes are "qcolorp vec4 compose(qcolorp vec4 src)":
@@ -321,7 +322,8 @@ public:
         ShockingPinkSrcFragmentShader,
 
         MaskFragmentShader,
-        RgbMaskFragmentShader,
+        RgbMaskFragmentShaderPass1,
+        RgbMaskFragmentShaderPass2,
         RgbMaskWithGammaFragmentShader,
 
         MultiplyCompositionModeFragmentShader,
@@ -376,7 +378,7 @@ public:
     QGLEngineShaderManager(QGLContext* context);
     ~QGLEngineShaderManager();
 
-    enum MaskType {NoMask, PixelMask, SubPixelMask, SubPixelWithGammaMask};
+    enum MaskType {NoMask, PixelMask, SubPixelMaskPass1, SubPixelMaskPass2, SubPixelWithGammaMask};
     enum PixelSrcType {
         ImageSrc = Qt::TexturePattern+1,
         NonPremultipliedImageSrc = Qt::TexturePattern+2,
diff --git a/src/opengl/gl2paintengineex/qglengineshadersource_p.h b/src/opengl/gl2paintengineex/qglengineshadersource_p.h
index 6bcf010..8ae86d3 100644
--- a/src/opengl/gl2paintengineex/qglengineshadersource_p.h
+++ b/src/opengl/gl2paintengineex/qglengineshadersource_p.h
@@ -401,6 +401,39 @@ static const char* const qglslMaskFragmentShader = "\
         return src * mask.a; \
     }";
 
+// For source over with subpixel antialiasing, the final color is calculated per component as follows
+// (.a is alpha component, .c is red, green or blue component):
+// alpha = src.a * mask.c * opacity
+// dest.c = dest.c * (1 - alpha) + src.c * alpha
+//
+// In the first pass, calculate: dest.c = dest.c * (1 - alpha) with blend funcs: zero, 1 - source color
+// In the second pass, calculate: dest.c = dest.c + src.c * alpha with blend funcs: one, one
+//
+// If source is a solid color (src is constant), only the first pass is needed, with blend funcs: constant, 1 - source color
+
+// For source composition with subpixel antialiasing, the final color is calculated per component as follows:
+// alpha = src.a * mask.c * opacity
+// dest.c = dest.c * (1 - mask.c) + src.c * alpha
+//
+
+static const char* const qglslRgbMaskFragmentShaderPass1 = "\
+    varying highp   vec2      textureCoords;\
+    uniform lowp    sampler2D maskTexture;\
+    lowp vec4 applyMask(lowp vec4 src) \
+    {\
+        lowp vec4 mask = texture2D(maskTexture, textureCoords); \
+        return src.a * mask; \
+    }";
+
+static const char* const qglslRgbMaskFragmentShaderPass2 = "\
+    varying highp   vec2      textureCoords;\
+    uniform lowp    sampler2D maskTexture;\
+    lowp vec4 applyMask(lowp vec4 src) \
+    {\
+        lowp vec4 mask = texture2D(maskTexture, textureCoords); \
+        return src * mask; \
+    }";
+
 /*
     Left to implement:
         RgbMaskFragmentShader,
diff --git a/src/opengl/gl2paintengineex/qpaintengineex_opengl2.cpp b/src/opengl/gl2paintengineex/qpaintengineex_opengl2.cpp
index b59328e..fb493e6 100644
--- a/src/opengl/gl2paintengineex/qpaintengineex_opengl2.cpp
+++ b/src/opengl/gl2paintengineex/qpaintengineex_opengl2.cpp
@@ -90,6 +90,10 @@ static const GLuint QT_IMAGE_TEXTURE_UNIT       = 0; //Can be the same as brush
 static const GLuint QT_MASK_TEXTURE_UNIT        = 1;
 static const GLuint QT_BACKGROUND_TEXTURE_UNIT  = 2;
 
+#ifdef Q_WS_WIN
+extern Q_GUI_EXPORT bool qt_cleartype_enabled;
+#endif
+
 class QGLTextureGlyphCache : public QObject, public QTextureGlyphCache
 {
     Q_OBJECT
@@ -100,6 +104,7 @@ public:
     virtual void createTextureData(int width, int height);
     virtual void resizeTextureData(int width, int height);
     virtual void fillTexture(const Coord &c, glyph_t glyph);
+    virtual int glyphMargin() const;
 
     inline GLuint texture() const { return m_texture; }
 
@@ -290,7 +295,7 @@ void QGLTextureGlyphCache::fillTexture(const Coord &c, glyph_t glyph)
 
     glBindTexture(GL_TEXTURE_2D, m_texture);
     if (mask.format() == QImage::Format_RGB32) {
-        glTexSubImage2D(GL_TEXTURE_2D, 0, c.x, m_height - c.y, maskWidth, maskHeight, GL_BGRA, GL_UNSIGNED_BYTE, mask.bits());
+        glTexSubImage2D(GL_TEXTURE_2D, 0, c.x, c.y, maskWidth, maskHeight, GL_BGRA, GL_UNSIGNED_BYTE, mask.bits());
     } else {
 #ifdef QT_OPENGL_ES2
         glTexSubImage2D(GL_TEXTURE_2D, 0, c.x, c.y, maskWidth, maskHeight, GL_ALPHA, GL_UNSIGNED_BYTE, mask.bits());
@@ -310,6 +315,17 @@ void QGLTextureGlyphCache::fillTexture(const Coord &c, glyph_t glyph)
     }
 }
 
+int QGLTextureGlyphCache::glyphMargin() const
+{
+#if defined(Q_WS_MAC)
+    return 2;
+#elif defined (Q_WS_X11)
+    return 0;
+#else
+    return m_type == QFontEngineGlyphCache::Raster_RGBMask ? 2 : 0;
+#endif
+}
+
 extern QImage qt_imageForBrush(int brushStyle, bool invert);
 
 ////////////////////////////////// Private Methods //////////////////////////////////////////
@@ -1210,6 +1226,12 @@ void QGL2PaintEngineEx::drawTextItem(const QPointF &p, const QTextItem &textItem
     if (ti.fontEngine->fontDef.pixelSize * qSqrt(s->matrix.determinant()) >= 64)
         drawCached = false;
 
+    if (d->glyphCacheType == QFontEngineGlyphCache::Raster_RGBMask
+        && state()->composition_mode != QPainter::CompositionMode_Source
+        && state()->composition_mode != QPainter::CompositionMode_SourceOver) {
+        drawCached = false;
+    }
+
     if (drawCached) {
         d->drawCachedGlyphs(p, ti);
         return;
@@ -1230,11 +1252,12 @@ void QGL2PaintEngineExPrivate::drawCachedGlyphs(const QPointF &p, const QTextIte
 
     QFontEngineGlyphCache::Type glyphType = ti.fontEngine->glyphFormat >= 0
         ? QFontEngineGlyphCache::Type(ti.fontEngine->glyphFormat)
-        : QFontEngineGlyphCache::Raster_A8;
+        : glyphCacheType;
 
     QGLTextureGlyphCache *cache =
         (QGLTextureGlyphCache *) ti.fontEngine->glyphCache(ctx, s->matrix);
-    if (!cache) {
+
+    if (!cache || cache->cacheType() != glyphType) {
         cache = new QGLTextureGlyphCache(ctx, glyphType, s->matrix);
         ti.fontEngine->setGlyphCache(ctx, cache);
     }
@@ -1249,12 +1272,6 @@ void QGL2PaintEngineExPrivate::drawCachedGlyphs(const QPointF &p, const QTextIte
         transferMode(BrushDrawingMode);
     transferMode(TextDrawingMode);
 
-    if (glyphType == QFontEngineGlyphCache::Raster_A8)
-        shaderManager->setMaskType(QGLEngineShaderManager::PixelMask);
-    else if (glyphType == QFontEngineGlyphCache::Raster_RGBMask)
-        shaderManager->setMaskType(QGLEngineShaderManager::SubPixelMask);
-    //### TODO: Gamma correction
-
     int margin = cache->glyphMargin();
 
     GLfloat dx = 1.0 / cache->width();
@@ -1275,10 +1292,96 @@ void QGL2PaintEngineExPrivate::drawCachedGlyphs(const QPointF &p, const QTextIte
         textureCoordinateArray.addRect(QRectF(c.x*dx, c.y*dy, c.w * dx, c.h * dy));
     }
 
+    if (vertexCoordinateArray.data() != oldVertexCoordinateDataPtr)
+        glVertexAttribPointer(QT_VERTEX_COORDS_ATTR, 2, GL_FLOAT, GL_FALSE, 0, vertexCoordinateArray.data());
+    if (textureCoordinateArray.data() != oldTextureCoordinateDataPtr)
+        glVertexAttribPointer(QT_TEXTURE_COORDS_ATTR, 2, GL_FLOAT, GL_FALSE, 0, textureCoordinateArray.data());
+
     QBrush pensBrush = q->state()->pen.brush();
     setBrush(&pensBrush);
 
-    prepareForDraw(false); // Text always causes src pixels to be transparent
+    if (glyphType == QFontEngineGlyphCache::Raster_A8) {
+
+        // Greyscale antialiasing
+
+        shaderManager->setMaskType(QGLEngineShaderManager::PixelMask);
+        prepareForDraw(false); // Text always causes src pixels to be transparent
+    } else if (glyphType == QFontEngineGlyphCache::Raster_RGBMask) {
+
+        // Subpixel antialiasing without gamma correction
+
+        QPainter::CompositionMode compMode = q->state()->composition_mode;
+        Q_ASSERT(compMode == QPainter::CompositionMode_Source
+            || compMode == QPainter::CompositionMode_SourceOver);
+
+        shaderManager->setMaskType(QGLEngineShaderManager::SubPixelMaskPass1);
+
+        if (pensBrush.style() == Qt::SolidPattern) {
+            // Solid patterns can get away with only one pass.
+            QColor c = pensBrush.color();
+            qreal oldOpacity = q->state()->opacity;
+            if (compMode == QPainter::CompositionMode_Source) {
+                c = premultiplyColor(c, q->state()->opacity);
+                q->state()->opacity = 1;
+                opacityUniformDirty = true;
+            }
+
+            compositionModeDirty = false; // I can handle this myself, thank you very much
+            prepareForDraw(false); // Text always causes src pixels to be transparent
+
+            // prepareForDraw() have set the opacity on the current shader, so the opacity state can now be reset.
+            if (compMode == QPainter::CompositionMode_Source) {
+                q->state()->opacity = oldOpacity;
+                opacityUniformDirty = true;
+            }
+
+            glEnable(GL_BLEND);
+            glBlendFunc(GL_CONSTANT_COLOR, GL_ONE_MINUS_SRC_COLOR);
+            glBlendColor(c.redF(), c.greenF(), c.blueF(), c.alphaF());
+        } else {
+            // Other brush styles need two passes.
+
+            qreal oldOpacity = q->state()->opacity;
+            if (compMode == QPainter::CompositionMode_Source) {
+                q->state()->opacity = 1;
+                opacityUniformDirty = true;
+                pensBrush = Qt::white;
+                setBrush(&pensBrush);
+            }
+
+            compositionModeDirty = false; // I can handle this myself, thank you very much
+            prepareForDraw(false); // Text always causes src pixels to be transparent
+            glEnable(GL_BLEND);
+            glBlendFunc(GL_ZERO, GL_ONE_MINUS_SRC_COLOR);
+
+            glActiveTexture(GL_TEXTURE0 + QT_MASK_TEXTURE_UNIT);
+            glBindTexture(GL_TEXTURE_2D, cache->texture());
+            updateTextureFilter(GL_TEXTURE_2D, GL_REPEAT, false);
+
+#ifndef QT_OPENGL_ES_2
+            if (inRenderText)
+                shaderManager->currentProgram()->setUniformValue(location(QGLEngineShaderManager::Depth), zValueForRenderText());
+#endif
+            shaderManager->currentProgram()->setUniformValue(location(QGLEngineShaderManager::MaskTexture), QT_MASK_TEXTURE_UNIT);
+            glDrawArrays(GL_TRIANGLES, 0, 6 * glyphs.size());
+
+            shaderManager->setMaskType(QGLEngineShaderManager::SubPixelMaskPass2);
+
+            if (compMode == QPainter::CompositionMode_Source) {
+                q->state()->opacity = oldOpacity;
+                opacityUniformDirty = true;
+                pensBrush = q->state()->pen.brush();
+                setBrush(&pensBrush);
+            }
+
+            compositionModeDirty = false;
+            prepareForDraw(false); // Text always causes src pixels to be transparent
+            glEnable(GL_BLEND);
+            glBlendFunc(GL_ONE, GL_ONE);
+        }
+        compositionModeDirty = true;
+    }
+    //### TODO: Gamma correction
 
     glActiveTexture(GL_TEXTURE0 + QT_MASK_TEXTURE_UNIT);
     glBindTexture(GL_TEXTURE_2D, cache->texture());
@@ -1288,14 +1391,7 @@ void QGL2PaintEngineExPrivate::drawCachedGlyphs(const QPointF &p, const QTextIte
     if (inRenderText)
         shaderManager->currentProgram()->setUniformValue(location(QGLEngineShaderManager::Depth), zValueForRenderText());
 #endif
-
     shaderManager->currentProgram()->setUniformValue(location(QGLEngineShaderManager::MaskTexture), QT_MASK_TEXTURE_UNIT);
-
-    if (vertexCoordinateArray.data() != oldVertexCoordinateDataPtr)
-        glVertexAttribPointer(QT_VERTEX_COORDS_ATTR, 2, GL_FLOAT, GL_FALSE, 0, vertexCoordinateArray.data());
-    if (textureCoordinateArray.data() != oldTextureCoordinateDataPtr)
-        glVertexAttribPointer(QT_TEXTURE_COORDS_ATTR, 2, GL_FLOAT, GL_FALSE, 0, textureCoordinateArray.data());
-
     glDrawArrays(GL_TRIANGLES, 0, 6 * glyphs.size());
 }
 
@@ -1355,6 +1451,18 @@ bool QGL2PaintEngineEx::begin(QPaintDevice *pdev)
     glDisable(GL_MULTISAMPLE);
 #endif
 
+    d->glyphCacheType = QFontEngineGlyphCache::Raster_A8;
+
+#if !defined(QT_OPENGL_ES_2)
+    if (!d->device->format().alpha()
+#if defined(Q_WS_WIN)
+        && qt_cleartype_enabled
+#endif
+       ) {
+        d->glyphCacheType = QFontEngineGlyphCache::Raster_RGBMask;
+    }
+#endif
+
     return true;
 }
 
diff --git a/src/opengl/gl2paintengineex/qpaintengineex_opengl2_p.h b/src/opengl/gl2paintengineex/qpaintengineex_opengl2_p.h
index 0b564b7..049994f 100644
--- a/src/opengl/gl2paintengineex/qpaintengineex_opengl2_p.h
+++ b/src/opengl/gl2paintengineex/qpaintengineex_opengl2_p.h
@@ -60,6 +60,7 @@
 #include <private/qgl2pexvertexarray_p.h>
 #include <private/qglpaintdevice_p.h>
 #include <private/qglpixmapfilter_p.h>
+#include <private/qfontengine_p.h>
 
 enum EngineMode {
     ImageDrawingMode,
@@ -205,6 +206,7 @@ public:
     int width, height;
     QGLContext *ctx;
     EngineMode mode;
+    QFontEngineGlyphCache::Type glyphCacheType;
 
     mutable QOpenGL2PaintEngineState *last_created_state;