Improved performance of 16 bit memrotates using NEON instructions.

Make the memrotate functions a function pointer table so that we can
replace it with optimized versions, and implement an optimized NEON
version for the 90 and 270 rotations.

Measured performance improvement for a 400x400 16-bit pixmap was
17 % for 270 degree rotation and 11 % for 90 degree rotation.

Reviewed-by: Trond

7 files changed, 310 insertions, 22 deletions

diff --git a/src/gui/painting/qdrawhelper.cpp b/src/gui/painting/qdrawhelper.cpp
index 5727b3c..d7f7dc3 100644
--- a/src/gui/painting/qdrawhelper.cpp
+++ b/src/gui/painting/qdrawhelper.cpp
@@ -7901,6 +7901,9 @@ void qInitDrawhelperAsm()
             functionForModeSolid_C[QPainter::CompositionMode_SourceOver] = comp_func_solid_SourceOver_neon;
             destFetchProc[QImage::Format_RGB16] = qt_destFetchRGB16_neon;
             destStoreProc[QImage::Format_RGB16] = qt_destStoreRGB16_neon;
+
+            qMemRotateFunctions[QImage::Format_RGB16][0] = qt_memrotate90_16_neon;
+            qMemRotateFunctions[QImage::Format_RGB16][2] = qt_memrotate270_16_neon;
         }
 #endif
 
diff --git a/src/gui/painting/qdrawhelper_neon.cpp b/src/gui/painting/qdrawhelper_neon.cpp
index 3ce90d2..03fe075 100644
--- a/src/gui/painting/qdrawhelper_neon.cpp
+++ b/src/gui/painting/qdrawhelper_neon.cpp
@@ -622,6 +622,150 @@ void QT_FASTCALL comp_func_solid_SourceOver_neon(uint *destPixels, int length, u
     }
 }
 
+static const int tileSize = 32;
+
+extern "C" void qt_rotate90_16_neon(quint16 *dst, const quint16 *src, int sstride, int dstride, int count);
+
+void qt_memrotate90_16_neon(const uchar *srcPixels, int w, int h, int sstride, uchar *destPixels, int dstride)
+{
+    const ushort *src = (const ushort *)srcPixels;
+    ushort *dest = (ushort *)destPixels;
+
+    sstride /= sizeof(ushort);
+    dstride /= sizeof(ushort);
+
+    const int pack = sizeof(quint32) / sizeof(ushort);
+    const int unaligned =
+        qMin(uint((quintptr(dest) & (sizeof(quint32)-1)) / sizeof(ushort)), uint(h));
+    const int restX = w % tileSize;
+    const int restY = (h - unaligned) % tileSize;
+    const int unoptimizedY = restY % pack;
+    const int numTilesX = w / tileSize + (restX > 0);
+    const int numTilesY = (h - unaligned) / tileSize + (restY >= pack);
+
+    for (int tx = 0; tx < numTilesX; ++tx) {
+        const int startx = w - tx * tileSize - 1;
+        const int stopx = qMax(startx - tileSize, 0);
+
+        if (unaligned) {
+            for (int x = startx; x >= stopx; --x) {
+                ushort *d = dest + (w - x - 1) * dstride;
+                for (int y = 0; y < unaligned; ++y) {
+                    *d++ = src[y * sstride + x];
+                }
+            }
+        }
+
+        for (int ty = 0; ty < numTilesY; ++ty) {
+            const int starty = ty * tileSize + unaligned;
+            const int stopy = qMin(starty + tileSize, h - unoptimizedY);
+
+            int x = startx;
+            // qt_rotate90_16_neon writes to eight rows, four pixels at a time
+            for (; x >= stopx + 7; x -= 8) {
+                ushort *d = dest + (w - x - 1) * dstride + starty;
+                const ushort *s = &src[starty * sstride + x - 7];
+                qt_rotate90_16_neon(d, s, sstride * 2, dstride * 2, stopy - starty);
+            }
+
+            for (; x >= stopx; --x) {
+                quint32 *d = reinterpret_cast<quint32*>(dest + (w - x - 1) * dstride + starty);
+                for (int y = starty; y < stopy; y += pack) {
+                    quint32 c = src[y * sstride + x];
+                    for (int i = 1; i < pack; ++i) {
+                        const int shift = (sizeof(int) * 8 / pack * i);
+                        const ushort color = src[(y + i) * sstride + x];
+                        c |= color << shift;
+                    }
+                    *d++ = c;
+                }
+            }
+        }
+
+        if (unoptimizedY) {
+            const int starty = h - unoptimizedY;
+            for (int x = startx; x >= stopx; --x) {
+                ushort *d = dest + (w - x - 1) * dstride + starty;
+                for (int y = starty; y < h; ++y) {
+                    *d++ = src[y * sstride + x];
+                }
+            }
+        }
+    }
+}
+
+extern "C" void qt_rotate270_16_neon(quint16 *dst, const quint16 *src, int sstride, int dstride, int count);
+
+void qt_memrotate270_16_neon(const uchar *srcPixels, int w, int h,
+                             int sstride,
+                             uchar *destPixels, int dstride)
+{
+    const ushort *src = (const ushort *)srcPixels;
+    ushort *dest = (ushort *)destPixels;
+
+    sstride /= sizeof(ushort);
+    dstride /= sizeof(ushort);
+
+    const int pack = sizeof(quint32) / sizeof(ushort);
+    const int unaligned =
+        qMin(uint((long(dest) & (sizeof(quint32)-1)) / sizeof(ushort)), uint(h));
+    const int restX = w % tileSize;
+    const int restY = (h - unaligned) % tileSize;
+    const int unoptimizedY = restY % pack;
+    const int numTilesX = w / tileSize + (restX > 0);
+    const int numTilesY = (h - unaligned) / tileSize + (restY >= pack);
+
+    for (int tx = 0; tx < numTilesX; ++tx) {
+        const int startx = tx * tileSize;
+        const int stopx = qMin(startx + tileSize, w);
+
+        if (unaligned) {
+            for (int x = startx; x < stopx; ++x) {
+                ushort *d = dest + x * dstride;
+                for (int y = h - 1; y >= h - unaligned; --y) {
+                    *d++ = src[y * sstride + x];
+                }
+            }
+        }
+
+        for (int ty = 0; ty < numTilesY; ++ty) {
+            const int starty = h - 1 - unaligned - ty * tileSize;
+            const int stopy = qMax(starty - tileSize, unoptimizedY);
+
+            int x = startx;
+            // qt_rotate90_16_neon writes to eight rows, four pixels at a time
+            for (; x < stopx - 7; x += 8) {
+                ushort *d = dest + x * dstride + h - 1 - starty;
+                const ushort *s = &src[starty * sstride + x];
+                qt_rotate90_16_neon(d + 7 * dstride, s, -sstride * 2, -dstride * 2, starty - stopy);
+            }
+
+            for (; x < stopx; ++x) {
+                quint32 *d = reinterpret_cast<quint32*>(dest + x * dstride
+                                                        + h - 1 - starty);
+                for (int y = starty; y > stopy; y -= pack) {
+                    quint32 c = src[y * sstride + x];
+                    for (int i = 1; i < pack; ++i) {
+                        const int shift = (sizeof(int) * 8 / pack * i);
+                        const ushort color = src[(y - i) * sstride + x];
+                        c |= color << shift;
+                    }
+                    *d++ = c;
+                }
+            }
+        }
+        if (unoptimizedY) {
+            const int starty = unoptimizedY - 1;
+            for (int x = startx; x < stopx; ++x) {
+                ushort *d = dest + x * dstride + h - 1 - starty;
+                for (int y = starty; y >= 0; --y) {
+                    *d++ = src[y * sstride + x];
+                }
+            }
+        }
+    }
+}
+
 QT_END_NAMESPACE
 
 #endif // QT_HAVE_NEON
diff --git a/src/gui/painting/qdrawhelper_neon_asm.S b/src/gui/painting/qdrawhelper_neon_asm.S
index 9992817..d9cdc36 100644
--- a/src/gui/painting/qdrawhelper_neon_asm.S
+++ b/src/gui/painting/qdrawhelper_neon_asm.S
@@ -190,3 +190,108 @@ blend_8_pixels_rgb16_on_rgb16_neon:
     bx          lr
 
     .endfunc
+
+/* void qt_rotate90_16_neon(quint16 *dst, const quint16 *src, int sstride, int dstride, int count) */
+    .func qt_rotate90_16_neon
+    .global qt_rotate90_16_neon
+    /* For ELF format also set function visibility to hidden */
+#ifdef __ELF__
+    .hidden qt_rotate90_16_neon
+    .type qt_rotate90_16_neon, %function
+#endif
+qt_rotate90_16_neon:
+    push { r4-r11, lr }
+    ldr r5, [sp, #(9*4)]
+
+    /* The preloads are the key to getting good performance */
+    pld [r1]
+
+    mov r4, r5, asr #2
+    add r6, r0, r3
+    add r7, r6, r3
+
+    add r8, r7, r3
+    add r9, r8, r3
+
+    pld [r1, r2]
+
+    add r10, r9, r3
+    add r11, r10, r3
+
+    add r3, r3, r11
+    and r5, r5, #3
+
+    pld [r1, r2, lsl #1]
+
+    cmp r4, #0
+    beq .rotate90_16_tail
+
+.rotate90_16_loop:
+    vld1.16 { q8  }, [r1], r2
+
+    pld [r1, r2, lsl #1]
+
+    vld1.16 { q9  }, [r1], r2
+    vld1.16 { q10 }, [r1], r2
+    vld1.16 { q11 }, [r1], r2
+
+    pld [r1]
+
+    /* Could have used four quad-word zips instead,
+       but those take three cycles as opposed to one. */
+    vzip.16 d16, d20
+    vzip.16 d17, d21
+
+    vzip.16 d18, d22
+
+    pld [r1, r2]
+
+    vzip.16 d19, d23
+
+    vzip.16 d16, d18
+    vzip.16 d17, d19
+
+    pld [r1, r2, lsl #1]
+
+    vzip.16 d20, d22
+    vzip.16 d21, d23
+
+    vst1.16 { d23 }, [r0]!
+    vst1.16 { d21 }, [r6]!
+    vst1.16 { d19 }, [r7]!
+    vst1.16 { d17 }, [r8]!
+    vst1.16 { d22 }, [r9]!
+    vst1.16 { d20 }, [r10]!
+    vst1.16 { d18 }, [r11]!
+    vst1.16 { d16 }, [r3]!
+
+    sub r4, r4, #1
+    cmp r4, #0
+    bne .rotate90_16_loop
+    b .rotate90_16_tail
+
+.rotate90_16_tail_loop:
+    sub r5, r5, #2
+
+    vld1.16 { q8 }, [r1], r2
+    vld1.16 { q9 }, [r1], r2
+
+    vzip.16 d16, d18
+    vzip.16 d17, d19
+
+    vst1.32 { d19[1] }, [r0]!
+    vst1.32 { d19[0] }, [r6]!
+    vst1.32 { d17[1] }, [r7]!
+    vst1.32 { d17[0] }, [r8]!
+    vst1.32 { d18[1] }, [r9]!
+    vst1.32 { d18[0] }, [r10]!
+    vst1.32 { d16[1] }, [r11]!
+    vst1.32 { d16[0] }, [r3]!
+
+.rotate90_16_tail:
+    cmp r5, #0
+    bgt .rotate90_16_tail_loop
+
+    pop { r4-r11, pc }
+
+    .endfunc
diff --git a/src/gui/painting/qdrawhelper_neon_p.h b/src/gui/painting/qdrawhelper_neon_p.h
index c054a1e..cd2dbfc 100644
--- a/src/gui/painting/qdrawhelper_neon_p.h
+++ b/src/gui/painting/qdrawhelper_neon_p.h
@@ -120,6 +120,9 @@ void qt_transform_image_rgb16_on_rgb16_neon(uchar *destPixels, int dbpl,
                                             const QTransform &targetRectTransform,
                                             int const_alpha);
 
+void qt_memrotate90_16_neon(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl);
+void qt_memrotate270_16_neon(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl);
+
 uint * QT_FASTCALL qt_destFetchRGB16_neon(uint *buffer,
                                           QRasterBuffer *rasterBuffer,
                                           int x, int y, int length);
diff --git a/src/gui/painting/qdrawhelper_p.h b/src/gui/painting/qdrawhelper_p.h
index acf765c..97c78bb 100644
--- a/src/gui/painting/qdrawhelper_p.h
+++ b/src/gui/painting/qdrawhelper_p.h
@@ -152,6 +152,7 @@ typedef void (*SrcOverTransformFunc)(uchar *destPixels, int dbpl,
                                      const QTransform &targetRectTransform,
                                      int const_alpha);
 
+typedef void (*MemRotateFunc)(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl);
 
 struct DrawHelper {
     ProcessSpans blendColor;
@@ -165,6 +166,7 @@ struct DrawHelper {
 extern SrcOverBlendFunc qBlendFunctions[QImage::NImageFormats][QImage::NImageFormats];
 extern SrcOverScaleFunc qScaleFunctions[QImage::NImageFormats][QImage::NImageFormats];
 extern SrcOverTransformFunc qTransformFunctions[QImage::NImageFormats][QImage::NImageFormats];
+extern MemRotateFunc qMemRotateFunctions[QImage::NImageFormats][3];
 
 extern DrawHelper qDrawHelper[QImage::NImageFormats];
 
diff --git a/src/gui/painting/qmemrotate.cpp b/src/gui/painting/qmemrotate.cpp
index c37aa51..6888bb0 100644
--- a/src/gui/painting/qmemrotate.cpp
+++ b/src/gui/painting/qmemrotate.cpp
@@ -594,4 +594,55 @@ void Q_GUI_EXPORT qt_memrotate90_gl(const quint32 *src, int srcWidth, int srcHei
     qt_memrotate90_template(src, srcWidth, srcHeight, srcStride, reinterpret_cast<qrgb_gl_rgba *>(dest), dstStride);
 }
 
+void qt_memrotate90_16(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl)
+{
+    qt_memrotate90((const ushort *)srcPixels, w, h, sbpl, (ushort *)destPixels, dbpl);
+}
+
+void qt_memrotate180_16(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl)
+{
+    qt_memrotate180((const ushort *)srcPixels, w, h, sbpl, (ushort *)destPixels, dbpl);
+}
+
+void qt_memrotate270_16(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl)
+{
+    qt_memrotate270((const ushort *)srcPixels, w, h, sbpl, (ushort *)destPixels, dbpl);
+}
+
+void qt_memrotate90_32(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl)
+{
+    qt_memrotate90((const uint *)srcPixels, w, h, sbpl, (uint *)destPixels, dbpl);
+}
+
+void qt_memrotate180_32(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl)
+{
+    qt_memrotate180((const uint *)srcPixels, w, h, sbpl, (uint *)destPixels, dbpl);
+}
+
+void qt_memrotate270_32(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl)
+{
+    qt_memrotate270((const uint *)srcPixels, w, h, sbpl, (uint *)destPixels, dbpl);
+}
+
+MemRotateFunc qMemRotateFunctions[QImage::NImageFormats][3] =
+// 90, 180, 270
+{
+    { 0, 0, 0 },      // Format_Invalid,
+    { 0, 0, 0 },      // Format_Mono,
+    { 0, 0, 0 },      // Format_MonoLSB,
+    { 0, 0, 0 },      // Format_Indexed8,
+    { qt_memrotate90_32, qt_memrotate180_32, qt_memrotate270_32 },      // Format_RGB32,
+    { qt_memrotate90_32, qt_memrotate180_32, qt_memrotate270_32 },      // Format_ARGB32,
+    { qt_memrotate90_32, qt_memrotate180_32, qt_memrotate270_32 },      // Format_ARGB32_Premultiplied,
+    { qt_memrotate90_16, qt_memrotate180_16, qt_memrotate270_16 },      // Format_RGB16,
+    { 0, 0, 0 },      // Format_ARGB8565_Premultiplied,
+    { 0, 0, 0 },      // Format_RGB666,
+    { 0, 0, 0 },      // Format_ARGB6666_Premultiplied,
+    { 0, 0, 0 },      // Format_RGB555,
+    { 0, 0, 0 },      // Format_ARGB8555_Premultiplied,
+    { 0, 0, 0 },      // Format_RGB888,
+    { 0, 0, 0 },      // Format_RGB444,
+    { 0, 0, 0 }       // Format_ARGB4444_Premultiplied,
+};
+
 QT_END_NAMESPACE
diff --git a/src/gui/painting/qpaintengine_raster.cpp b/src/gui/painting/qpaintengine_raster.cpp
index 84b36c7..09a87aa 100644
--- a/src/gui/painting/qpaintengine_raster.cpp
+++ b/src/gui/painting/qpaintengine_raster.cpp
@@ -2553,23 +2553,6 @@ namespace {
         return NoRotation;
     }
 
-    template <typename T> void memRotate(RotationType type, const T *srcBase, int w, int h, int sbpl, T *dstBase, int dbpl)
-    {
-        switch (type) {
-        case Rotation90:
-            qt_memrotate90(srcBase, w, h, sbpl, dstBase, dbpl);
-            break;
-        case Rotation180:
-            qt_memrotate180(srcBase, w, h, sbpl, dstBase, dbpl);
-            break;
-        case Rotation270:
-            qt_memrotate270(srcBase, w, h, sbpl, dstBase, dbpl);
-            break;
-        case NoRotation:
-            break;
-        }
-    }
-
     inline bool isPixelAligned(const QRectF &rect) {
         return QRectF(rect.toRect()) == rect;
     }
@@ -2650,7 +2633,7 @@ void QRasterPaintEngine::drawImage(const QRectF &r, const QImage &img, const QRe
     {
         RotationType rotationType = qRotationType(s->matrix);
 
-        if (rotationType != NoRotation && img.rect().contains(sr.toAlignedRect())) {
+        if (rotationType != NoRotation && qMemRotateFunctions[d->rasterBuffer->format][rotationType] && img.rect().contains(sr.toAlignedRect())) {
             QRectF transformedTargetRect = s->matrix.mapRect(r);
 
             if ((!(s->renderHints & QPainter::SmoothPixmapTransform) && !(s->renderHints & QPainter::Antialiasing))
@@ -2678,10 +2661,7 @@ void QRasterPaintEngine::drawImage(const QRectF &r, const QImage &img, const QRe
                 uint cw = clippedSourceRect.width();
                 uint ch = clippedSourceRect.height();
 
-                if (d->rasterBuffer->format == QImage::Format_RGB16)
-                    memRotate(rotationType, (quint16 *)srcBase, cw, ch, sbpl, (quint16 *)dstBase, dbpl);
-                else
-                    memRotate(rotationType, (quint32 *)srcBase, cw, ch, sbpl, (quint32 *)dstBase, dbpl);
+                qMemRotateFunctions[d->rasterBuffer->format][rotationType](srcBase, cw, ch, sbpl, dstBase, dbpl);
 
                 return;
             }