From e302b2b1e65c616d7ed6adfcf4b252a6741f3053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samuel=20R=C3=B8dal?= Date: Wed, 30 Jun 2010 10:32:01 +0200 Subject: Improved performance of 16 bit memrotates using NEON instructions. Make the memrotate functions a function pointer table so that we can replace it with optimized versions, and implement an optimized NEON version for the 90 and 270 rotations. Measured performance improvement for a 400x400 16-bit pixmap was 17 % for 270 degree rotation and 11 % for 90 degree rotation. Reviewed-by: Trond --- src/gui/painting/qdrawhelper.cpp | 3 + src/gui/painting/qdrawhelper_neon.cpp | 144 +++++++++++++++++++++++++++++++ src/gui/painting/qdrawhelper_neon_asm.S | 105 ++++++++++++++++++++++ src/gui/painting/qdrawhelper_neon_p.h | 3 + src/gui/painting/qdrawhelper_p.h | 2 + src/gui/painting/qmemrotate.cpp | 51 +++++++++++ src/gui/painting/qpaintengine_raster.cpp | 24 +----- 7 files changed, 310 insertions(+), 22 deletions(-) diff --git a/src/gui/painting/qdrawhelper.cpp b/src/gui/painting/qdrawhelper.cpp index 5727b3c..d7f7dc3 100644 --- a/src/gui/painting/qdrawhelper.cpp +++ b/src/gui/painting/qdrawhelper.cpp @@ -7901,6 +7901,9 @@ void qInitDrawhelperAsm() functionForModeSolid_C[QPainter::CompositionMode_SourceOver] = comp_func_solid_SourceOver_neon; destFetchProc[QImage::Format_RGB16] = qt_destFetchRGB16_neon; destStoreProc[QImage::Format_RGB16] = qt_destStoreRGB16_neon; + + qMemRotateFunctions[QImage::Format_RGB16][0] = qt_memrotate90_16_neon; + qMemRotateFunctions[QImage::Format_RGB16][2] = qt_memrotate270_16_neon; } #endif diff --git a/src/gui/painting/qdrawhelper_neon.cpp b/src/gui/painting/qdrawhelper_neon.cpp index 3ce90d2..03fe075 100644 --- a/src/gui/painting/qdrawhelper_neon.cpp +++ b/src/gui/painting/qdrawhelper_neon.cpp @@ -622,6 +622,150 @@ void QT_FASTCALL comp_func_solid_SourceOver_neon(uint *destPixels, int length, u } } +static const int tileSize = 32; + +extern "C" void qt_rotate90_16_neon(quint16 *dst, const quint16 *src, int sstride, int dstride, int count); + +void qt_memrotate90_16_neon(const uchar *srcPixels, int w, int h, int sstride, uchar *destPixels, int dstride) +{ + const ushort *src = (const ushort *)srcPixels; + ushort *dest = (ushort *)destPixels; + + sstride /= sizeof(ushort); + dstride /= sizeof(ushort); + + const int pack = sizeof(quint32) / sizeof(ushort); + const int unaligned = + qMin(uint((quintptr(dest) & (sizeof(quint32)-1)) / sizeof(ushort)), uint(h)); + const int restX = w % tileSize; + const int restY = (h - unaligned) % tileSize; + const int unoptimizedY = restY % pack; + const int numTilesX = w / tileSize + (restX > 0); + const int numTilesY = (h - unaligned) / tileSize + (restY >= pack); + + for (int tx = 0; tx < numTilesX; ++tx) { + const int startx = w - tx * tileSize - 1; + const int stopx = qMax(startx - tileSize, 0); + + if (unaligned) { + for (int x = startx; x >= stopx; --x) { + ushort *d = dest + (w - x - 1) * dstride; + for (int y = 0; y < unaligned; ++y) { + *d++ = src[y * sstride + x]; + } + } + } + + for (int ty = 0; ty < numTilesY; ++ty) { + const int starty = ty * tileSize + unaligned; + const int stopy = qMin(starty + tileSize, h - unoptimizedY); + + int x = startx; + // qt_rotate90_16_neon writes to eight rows, four pixels at a time + for (; x >= stopx + 7; x -= 8) { + ushort *d = dest + (w - x - 1) * dstride + starty; + const ushort *s = &src[starty * sstride + x - 7]; + qt_rotate90_16_neon(d, s, sstride * 2, dstride * 2, stopy - starty); + } + + for (; x >= stopx; --x) { + quint32 *d = reinterpret_cast(dest + (w - x - 1) * dstride + starty); + for (int y = starty; y < stopy; y += pack) { + quint32 c = src[y * sstride + x]; + for (int i = 1; i < pack; ++i) { + const int shift = (sizeof(int) * 8 / pack * i); + const ushort color = src[(y + i) * sstride + x]; + c |= color << shift; + } + *d++ = c; + } + } + } + + if (unoptimizedY) { + const int starty = h - unoptimizedY; + for (int x = startx; x >= stopx; --x) { + ushort *d = dest + (w - x - 1) * dstride + starty; + for (int y = starty; y < h; ++y) { + *d++ = src[y * sstride + x]; + } + } + } + } +} + +extern "C" void qt_rotate270_16_neon(quint16 *dst, const quint16 *src, int sstride, int dstride, int count); + +void qt_memrotate270_16_neon(const uchar *srcPixels, int w, int h, + int sstride, + uchar *destPixels, int dstride) +{ + const ushort *src = (const ushort *)srcPixels; + ushort *dest = (ushort *)destPixels; + + sstride /= sizeof(ushort); + dstride /= sizeof(ushort); + + const int pack = sizeof(quint32) / sizeof(ushort); + const int unaligned = + qMin(uint((long(dest) & (sizeof(quint32)-1)) / sizeof(ushort)), uint(h)); + const int restX = w % tileSize; + const int restY = (h - unaligned) % tileSize; + const int unoptimizedY = restY % pack; + const int numTilesX = w / tileSize + (restX > 0); + const int numTilesY = (h - unaligned) / tileSize + (restY >= pack); + + for (int tx = 0; tx < numTilesX; ++tx) { + const int startx = tx * tileSize; + const int stopx = qMin(startx + tileSize, w); + + if (unaligned) { + for (int x = startx; x < stopx; ++x) { + ushort *d = dest + x * dstride; + for (int y = h - 1; y >= h - unaligned; --y) { + *d++ = src[y * sstride + x]; + } + } + } + + for (int ty = 0; ty < numTilesY; ++ty) { + const int starty = h - 1 - unaligned - ty * tileSize; + const int stopy = qMax(starty - tileSize, unoptimizedY); + + int x = startx; + // qt_rotate90_16_neon writes to eight rows, four pixels at a time + for (; x < stopx - 7; x += 8) { + ushort *d = dest + x * dstride + h - 1 - starty; + const ushort *s = &src[starty * sstride + x]; + qt_rotate90_16_neon(d + 7 * dstride, s, -sstride * 2, -dstride * 2, starty - stopy); + } + + for (; x < stopx; ++x) { + quint32 *d = reinterpret_cast(dest + x * dstride + + h - 1 - starty); + for (int y = starty; y > stopy; y -= pack) { + quint32 c = src[y * sstride + x]; + for (int i = 1; i < pack; ++i) { + const int shift = (sizeof(int) * 8 / pack * i); + const ushort color = src[(y - i) * sstride + x]; + c |= color << shift; + } + *d++ = c; + } + } + } + if (unoptimizedY) { + const int starty = unoptimizedY - 1; + for (int x = startx; x < stopx; ++x) { + ushort *d = dest + x * dstride + h - 1 - starty; + for (int y = starty; y >= 0; --y) { + *d++ = src[y * sstride + x]; + } + } + } + } +} + QT_END_NAMESPACE #endif // QT_HAVE_NEON diff --git a/src/gui/painting/qdrawhelper_neon_asm.S b/src/gui/painting/qdrawhelper_neon_asm.S index 9992817..d9cdc36 100644 --- a/src/gui/painting/qdrawhelper_neon_asm.S +++ b/src/gui/painting/qdrawhelper_neon_asm.S @@ -190,3 +190,108 @@ blend_8_pixels_rgb16_on_rgb16_neon: bx lr .endfunc + +/* void qt_rotate90_16_neon(quint16 *dst, const quint16 *src, int sstride, int dstride, int count) */ + .func qt_rotate90_16_neon + .global qt_rotate90_16_neon + /* For ELF format also set function visibility to hidden */ +#ifdef __ELF__ + .hidden qt_rotate90_16_neon + .type qt_rotate90_16_neon, %function +#endif +qt_rotate90_16_neon: + push { r4-r11, lr } + ldr r5, [sp, #(9*4)] + + /* The preloads are the key to getting good performance */ + pld [r1] + + mov r4, r5, asr #2 + add r6, r0, r3 + add r7, r6, r3 + + add r8, r7, r3 + add r9, r8, r3 + + pld [r1, r2] + + add r10, r9, r3 + add r11, r10, r3 + + add r3, r3, r11 + and r5, r5, #3 + + pld [r1, r2, lsl #1] + + cmp r4, #0 + beq .rotate90_16_tail + +.rotate90_16_loop: + vld1.16 { q8 }, [r1], r2 + + pld [r1, r2, lsl #1] + + vld1.16 { q9 }, [r1], r2 + vld1.16 { q10 }, [r1], r2 + vld1.16 { q11 }, [r1], r2 + + pld [r1] + + /* Could have used four quad-word zips instead, + but those take three cycles as opposed to one. */ + vzip.16 d16, d20 + vzip.16 d17, d21 + + vzip.16 d18, d22 + + pld [r1, r2] + + vzip.16 d19, d23 + + vzip.16 d16, d18 + vzip.16 d17, d19 + + pld [r1, r2, lsl #1] + + vzip.16 d20, d22 + vzip.16 d21, d23 + + vst1.16 { d23 }, [r0]! + vst1.16 { d21 }, [r6]! + vst1.16 { d19 }, [r7]! + vst1.16 { d17 }, [r8]! + vst1.16 { d22 }, [r9]! + vst1.16 { d20 }, [r10]! + vst1.16 { d18 }, [r11]! + vst1.16 { d16 }, [r3]! + + sub r4, r4, #1 + cmp r4, #0 + bne .rotate90_16_loop + b .rotate90_16_tail + +.rotate90_16_tail_loop: + sub r5, r5, #2 + + vld1.16 { q8 }, [r1], r2 + vld1.16 { q9 }, [r1], r2 + + vzip.16 d16, d18 + vzip.16 d17, d19 + + vst1.32 { d19[1] }, [r0]! + vst1.32 { d19[0] }, [r6]! + vst1.32 { d17[1] }, [r7]! + vst1.32 { d17[0] }, [r8]! + vst1.32 { d18[1] }, [r9]! + vst1.32 { d18[0] }, [r10]! + vst1.32 { d16[1] }, [r11]! + vst1.32 { d16[0] }, [r3]! + +.rotate90_16_tail: + cmp r5, #0 + bgt .rotate90_16_tail_loop + + pop { r4-r11, pc } + + .endfunc diff --git a/src/gui/painting/qdrawhelper_neon_p.h b/src/gui/painting/qdrawhelper_neon_p.h index c054a1e..cd2dbfc 100644 --- a/src/gui/painting/qdrawhelper_neon_p.h +++ b/src/gui/painting/qdrawhelper_neon_p.h @@ -120,6 +120,9 @@ void qt_transform_image_rgb16_on_rgb16_neon(uchar *destPixels, int dbpl, const QTransform &targetRectTransform, int const_alpha); +void qt_memrotate90_16_neon(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl); +void qt_memrotate270_16_neon(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl); + uint * QT_FASTCALL qt_destFetchRGB16_neon(uint *buffer, QRasterBuffer *rasterBuffer, int x, int y, int length); diff --git a/src/gui/painting/qdrawhelper_p.h b/src/gui/painting/qdrawhelper_p.h index acf765c..97c78bb 100644 --- a/src/gui/painting/qdrawhelper_p.h +++ b/src/gui/painting/qdrawhelper_p.h @@ -152,6 +152,7 @@ typedef void (*SrcOverTransformFunc)(uchar *destPixels, int dbpl, const QTransform &targetRectTransform, int const_alpha); +typedef void (*MemRotateFunc)(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl); struct DrawHelper { ProcessSpans blendColor; @@ -165,6 +166,7 @@ struct DrawHelper { extern SrcOverBlendFunc qBlendFunctions[QImage::NImageFormats][QImage::NImageFormats]; extern SrcOverScaleFunc qScaleFunctions[QImage::NImageFormats][QImage::NImageFormats]; extern SrcOverTransformFunc qTransformFunctions[QImage::NImageFormats][QImage::NImageFormats]; +extern MemRotateFunc qMemRotateFunctions[QImage::NImageFormats][3]; extern DrawHelper qDrawHelper[QImage::NImageFormats]; diff --git a/src/gui/painting/qmemrotate.cpp b/src/gui/painting/qmemrotate.cpp index c37aa51..6888bb0 100644 --- a/src/gui/painting/qmemrotate.cpp +++ b/src/gui/painting/qmemrotate.cpp @@ -594,4 +594,55 @@ void Q_GUI_EXPORT qt_memrotate90_gl(const quint32 *src, int srcWidth, int srcHei qt_memrotate90_template(src, srcWidth, srcHeight, srcStride, reinterpret_cast(dest), dstStride); } +void qt_memrotate90_16(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl) +{ + qt_memrotate90((const ushort *)srcPixels, w, h, sbpl, (ushort *)destPixels, dbpl); +} + +void qt_memrotate180_16(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl) +{ + qt_memrotate180((const ushort *)srcPixels, w, h, sbpl, (ushort *)destPixels, dbpl); +} + +void qt_memrotate270_16(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl) +{ + qt_memrotate270((const ushort *)srcPixels, w, h, sbpl, (ushort *)destPixels, dbpl); +} + +void qt_memrotate90_32(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl) +{ + qt_memrotate90((const uint *)srcPixels, w, h, sbpl, (uint *)destPixels, dbpl); +} + +void qt_memrotate180_32(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl) +{ + qt_memrotate180((const uint *)srcPixels, w, h, sbpl, (uint *)destPixels, dbpl); +} + +void qt_memrotate270_32(const uchar *srcPixels, int w, int h, int sbpl, uchar *destPixels, int dbpl) +{ + qt_memrotate270((const uint *)srcPixels, w, h, sbpl, (uint *)destPixels, dbpl); +} + +MemRotateFunc qMemRotateFunctions[QImage::NImageFormats][3] = +// 90, 180, 270 +{ + { 0, 0, 0 }, // Format_Invalid, + { 0, 0, 0 }, // Format_Mono, + { 0, 0, 0 }, // Format_MonoLSB, + { 0, 0, 0 }, // Format_Indexed8, + { qt_memrotate90_32, qt_memrotate180_32, qt_memrotate270_32 }, // Format_RGB32, + { qt_memrotate90_32, qt_memrotate180_32, qt_memrotate270_32 }, // Format_ARGB32, + { qt_memrotate90_32, qt_memrotate180_32, qt_memrotate270_32 }, // Format_ARGB32_Premultiplied, + { qt_memrotate90_16, qt_memrotate180_16, qt_memrotate270_16 }, // Format_RGB16, + { 0, 0, 0 }, // Format_ARGB8565_Premultiplied, + { 0, 0, 0 }, // Format_RGB666, + { 0, 0, 0 }, // Format_ARGB6666_Premultiplied, + { 0, 0, 0 }, // Format_RGB555, + { 0, 0, 0 }, // Format_ARGB8555_Premultiplied, + { 0, 0, 0 }, // Format_RGB888, + { 0, 0, 0 }, // Format_RGB444, + { 0, 0, 0 } // Format_ARGB4444_Premultiplied, +}; + QT_END_NAMESPACE diff --git a/src/gui/painting/qpaintengine_raster.cpp b/src/gui/painting/qpaintengine_raster.cpp index 84b36c7..09a87aa 100644 --- a/src/gui/painting/qpaintengine_raster.cpp +++ b/src/gui/painting/qpaintengine_raster.cpp @@ -2553,23 +2553,6 @@ namespace { return NoRotation; } - template void memRotate(RotationType type, const T *srcBase, int w, int h, int sbpl, T *dstBase, int dbpl) - { - switch (type) { - case Rotation90: - qt_memrotate90(srcBase, w, h, sbpl, dstBase, dbpl); - break; - case Rotation180: - qt_memrotate180(srcBase, w, h, sbpl, dstBase, dbpl); - break; - case Rotation270: - qt_memrotate270(srcBase, w, h, sbpl, dstBase, dbpl); - break; - case NoRotation: - break; - } - } - inline bool isPixelAligned(const QRectF &rect) { return QRectF(rect.toRect()) == rect; } @@ -2650,7 +2633,7 @@ void QRasterPaintEngine::drawImage(const QRectF &r, const QImage &img, const QRe { RotationType rotationType = qRotationType(s->matrix); - if (rotationType != NoRotation && img.rect().contains(sr.toAlignedRect())) { + if (rotationType != NoRotation && qMemRotateFunctions[d->rasterBuffer->format][rotationType] && img.rect().contains(sr.toAlignedRect())) { QRectF transformedTargetRect = s->matrix.mapRect(r); if ((!(s->renderHints & QPainter::SmoothPixmapTransform) && !(s->renderHints & QPainter::Antialiasing)) @@ -2678,10 +2661,7 @@ void QRasterPaintEngine::drawImage(const QRectF &r, const QImage &img, const QRe uint cw = clippedSourceRect.width(); uint ch = clippedSourceRect.height(); - if (d->rasterBuffer->format == QImage::Format_RGB16) - memRotate(rotationType, (quint16 *)srcBase, cw, ch, sbpl, (quint16 *)dstBase, dbpl); - else - memRotate(rotationType, (quint32 *)srcBase, cw, ch, sbpl, (quint32 *)dstBase, dbpl); + qMemRotateFunctions[d->rasterBuffer->format][rotationType](srcBase, cw, ch, sbpl, dstBase, dbpl); return; } -- cgit v0.12 ="hl com"> combined with a lookup table of jump addresses. However, since the indirect jump instruction is shared by all opcodes, the CPU will have a hard time making the right prediction for where to jump next (actually, it will be always wrong except in the uncommon case of a sequence of several identical opcodes). "Threaded code" in contrast, uses an explicit jump table and an explicit indirect jump instruction at the end of each opcode. Since the jump instruction is at a different address for each opcode, the CPU will make a separate prediction for each of these instructions, which is equivalent to predicting the second opcode of each opcode pair. These predictions have a much better chance to turn out valid, especially in small bytecode loops. A mispredicted branch on a modern CPU flushes the whole pipeline and can cost several CPU cycles (depending on the pipeline depth), and potentially many more instructions (depending on the pipeline width). A correctly predicted branch, however, is nearly free. At the time of this writing, the "threaded code" version is up to 15-20% faster than the normal "switch" version, depending on the compiler and the CPU architecture. NOTE: care must be taken that the compiler doesn't try to "optimize" the indirect jumps by sharing them between all opcodes. Such optimizations can be disabled on gcc by using the -fno-gcse flag (or possibly -fno-crossjumping). */ /* Use macros rather than inline functions, to make it as clear as possible * to the C compiler that the tracing check is a simple test then branch. * We want to be sure that the compiler knows this before it generates * the CFG. */ #ifdef WITH_DTRACE #define OR_DTRACE_LINE | (PyDTrace_LINE_ENABLED() ? 255 : 0) #else #define OR_DTRACE_LINE #endif #ifdef HAVE_COMPUTED_GOTOS #ifndef USE_COMPUTED_GOTOS #define USE_COMPUTED_GOTOS 1 #endif #else #if defined(USE_COMPUTED_GOTOS) && USE_COMPUTED_GOTOS #error "Computed gotos are not supported on this compiler." #endif #undef USE_COMPUTED_GOTOS #define USE_COMPUTED_GOTOS 0 #endif #ifdef Py_STATS #define INSTRUCTION_STATS(op) \ do { \ OPCODE_EXE_INC(op); \ if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \ lastopcode = op; \ } while (0) #else #define INSTRUCTION_STATS(op) ((void)0) #endif #if USE_COMPUTED_GOTOS # define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] #else # define TARGET(op) case op: TARGET_##op: # define DISPATCH_GOTO() goto dispatch_opcode #endif /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef LLTRACE #define PRE_DISPATCH_GOTO() if (lltrace >= 5) { \ lltrace_instruction(frame, stack_pointer, next_instr, opcode, oparg); } #else #define PRE_DISPATCH_GOTO() ((void)0) #endif /* Do interpreter dispatch accounting for tracing and instrumentation */ #define DISPATCH() \ { \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ } #define DISPATCH_SAME_OPARG() \ { \ opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ } #define DISPATCH_INLINED(NEW_FRAME) \ do { \ assert(tstate->interp->eval_frame == NULL); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ (NEW_FRAME)->previous = frame; \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ goto start_frame; \ } while (0) // Use this instead of 'goto error' so Tier 2 can go to a different label #define GOTO_ERROR(LABEL) goto LABEL #define CHECK_EVAL_BREAKER() \ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \ if (_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & _PY_EVAL_EVENTS_MASK) { \ if (_Py_HandlePending(tstate) != 0) { \ GOTO_ERROR(error); \ } \ } /* Tuple access macros */ #ifndef Py_DEBUG #define GETITEM(v, i) PyTuple_GET_ITEM((v), (i)) #else static inline PyObject * GETITEM(PyObject *v, Py_ssize_t i) { assert(PyTuple_Check(v)); assert(i >= 0); assert(i < PyTuple_GET_SIZE(v)); return PyTuple_GET_ITEM(v, i); } #endif /* Code access macros */ /* The integer overflow is checked by an assertion below. */ #define INSTR_OFFSET() ((int)(next_instr - _PyCode_CODE(_PyFrame_GetCode(frame)))) #define NEXTOPARG() do { \ _Py_CODEUNIT word = *next_instr; \ opcode = word.op.code; \ oparg = word.op.arg; \ } while (0) /* JUMPBY makes the generator identify the instruction as a jump. SKIP_OVER is * for advancing to the next instruction, taking into account cache entries * and skipped instructions. */ #define JUMPBY(x) (next_instr += (x)) #define SKIP_OVER(x) (next_instr += (x)) /* OpCode prediction macros Some opcodes tend to come in pairs thus making it possible to predict the second code when the first is run. For example, COMPARE_OP is often followed by POP_JUMP_IF_FALSE or POP_JUMP_IF_TRUE. Verifying the prediction costs a single high-speed test of a register variable against a constant. If the pairing was good, then the processor's own internal branch predication has a high likelihood of success, resulting in a nearly zero-overhead transition to the next opcode. A successful prediction saves a trip through the eval-loop including its unpredictable switch-case branch. Combined with the processor's internal branch prediction, a successful PREDICT has the effect of making the two opcodes run as if they were a single new opcode with the bodies combined. If collecting opcode statistics, your choices are to either keep the predictions turned-on and interpret the results as if some opcodes had been combined or turn-off predictions so that the opcode frequency counter updates for both opcodes. Opcode prediction is disabled with threaded code, since the latter allows the CPU to record separate branch prediction information for each opcode. */ #define PREDICT_ID(op) PRED_##op #define PREDICTED(op) PREDICT_ID(op): /* Stack manipulation macros */ /* The stack can grow at most MAXINT deep, as co_nlocals and co_stacksize are ints. */ #define STACK_LEVEL() ((int)(stack_pointer - _PyFrame_Stackbase(frame))) #define STACK_SIZE() (_PyFrame_GetCode(frame)->co_stacksize) #define EMPTY() (STACK_LEVEL() == 0) #define TOP() (stack_pointer[-1]) #define SECOND() (stack_pointer[-2]) #define THIRD() (stack_pointer[-3]) #define FOURTH() (stack_pointer[-4]) #define PEEK(n) (stack_pointer[-(n)]) #define POKE(n, v) (stack_pointer[-(n)] = (v)) #define SET_TOP(v) (stack_pointer[-1] = (v)) #define SET_SECOND(v) (stack_pointer[-2] = (v)) #define BASIC_STACKADJ(n) (stack_pointer += n) #define BASIC_PUSH(v) (*stack_pointer++ = (v)) #define BASIC_POP() (*--stack_pointer) #ifdef Py_DEBUG #define PUSH(v) do { \ BASIC_PUSH(v); \ assert(STACK_LEVEL() <= STACK_SIZE()); \ } while (0) #define POP() (assert(STACK_LEVEL() > 0), BASIC_POP()) #define STACK_GROW(n) do { \ assert(n >= 0); \ BASIC_STACKADJ(n); \ assert(STACK_LEVEL() <= STACK_SIZE()); \ } while (0) #define STACK_SHRINK(n) do { \ assert(n >= 0); \ assert(STACK_LEVEL() >= n); \ BASIC_STACKADJ(-(n)); \ } while (0) #else #define PUSH(v) BASIC_PUSH(v) #define POP() BASIC_POP() #define STACK_GROW(n) BASIC_STACKADJ(n) #define STACK_SHRINK(n) BASIC_STACKADJ(-(n)) #endif /* Data access macros */ #define FRAME_CO_CONSTS (_PyFrame_GetCode(frame)->co_consts) #define FRAME_CO_NAMES (_PyFrame_GetCode(frame)->co_names) /* Local variable macros */ #define LOCALS_ARRAY (frame->localsplus) #define GETLOCAL(i) (frame->localsplus[i]) /* The SETLOCAL() macro must not DECREF the local variable in-place and then store the new value; it must copy the old value to a temporary value, then store the new value, and then DECREF the temporary value. This is because it is possible that during the DECREF the frame is accessed by other code (e.g. a __del__ method or gc.collect()) and the variable would be pointing to already-freed memory. */ #define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i); \ GETLOCAL(i) = value; \ Py_XDECREF(tmp); } while (0) #define GO_TO_INSTRUCTION(op) goto PREDICT_ID(op) #ifdef Py_STATS #define UPDATE_MISS_STATS(INSTNAME) \ do { \ STAT_INC(opcode, miss); \ STAT_INC((INSTNAME), miss); \ /* The counter is always the first cache entry: */ \ if (ADAPTIVE_COUNTER_IS_ZERO(next_instr->cache)) { \ STAT_INC((INSTNAME), deopt); \ } \ } while (0) #else #define UPDATE_MISS_STATS(INSTNAME) ((void)0) #endif #define DEOPT_IF(COND, INSTNAME) \ if ((COND)) { \ /* This is only a single jump on release builds! */ \ UPDATE_MISS_STATS((INSTNAME)); \ assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ GO_TO_INSTRUCTION(INSTNAME); \ } #define GLOBALS() frame->f_globals #define BUILTINS() frame->f_builtins #define LOCALS() frame->f_locals #define CONSTS() _PyFrame_GetCode(frame)->co_consts #define NAMES() _PyFrame_GetCode(frame)->co_names #define DTRACE_FUNCTION_ENTRY() \ if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \ dtrace_function_entry(frame); \ } #define ADAPTIVE_COUNTER_IS_ZERO(COUNTER) \ (((COUNTER) >> ADAPTIVE_BACKOFF_BITS) == 0) #define ADAPTIVE_COUNTER_IS_MAX(COUNTER) \ (((COUNTER) >> ADAPTIVE_BACKOFF_BITS) == ((1 << MAX_BACKOFF_VALUE) - 1)) #define DECREMENT_ADAPTIVE_COUNTER(COUNTER) \ do { \ assert(!ADAPTIVE_COUNTER_IS_ZERO((COUNTER))); \ (COUNTER) -= (1 << ADAPTIVE_BACKOFF_BITS); \ } while (0); #define INCREMENT_ADAPTIVE_COUNTER(COUNTER) \ do { \ (COUNTER) += (1 << ADAPTIVE_BACKOFF_BITS); \ } while (0); #define UNBOUNDLOCAL_ERROR_MSG \ "cannot access local variable '%s' where it is not associated with a value" #define UNBOUNDFREE_ERROR_MSG \ "cannot access free variable '%s' where it is not associated with a value" \ " in enclosing scope" #define NAME_ERROR_MSG "name '%.200s' is not defined" #define DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dval, result) \ do { \ if (Py_REFCNT(left) == 1) { \ ((PyFloatObject *)left)->ob_fval = (dval); \ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);\ result = (left); \ } \ else if (Py_REFCNT(right) == 1) {\ ((PyFloatObject *)right)->ob_fval = (dval); \ _Py_DECREF_NO_DEALLOC(left); \ result = (right); \ }\ else { \ result = PyFloat_FromDouble(dval); \ if ((result) == NULL) GOTO_ERROR(error); \ _Py_DECREF_NO_DEALLOC(left); \ _Py_DECREF_NO_DEALLOC(right); \ } \ } while (0) // If a trace function sets a new f_lineno and // *then* raises, we use the destination when searching // for an exception handler, displaying the traceback, and so on #define INSTRUMENTED_JUMP(src, dest, event) \ do { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ next_instr = (dest)+1; \ goto error; \ } \ } while (0); typedef PyObject *(*convertion_func_ptr)(PyObject *); static const convertion_func_ptr CONVERSION_FUNCTIONS[4] = { [FVC_STR] = PyObject_Str, [FVC_REPR] = PyObject_Repr, [FVC_ASCII] = PyObject_ASCII }; // GH-89279: Force inlining by using a macro. #if defined(_MSC_VER) && SIZEOF_INT == 4 #define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) (assert(sizeof((ATOMIC_VAL)->_value) == 4), *((volatile int*)&((ATOMIC_VAL)->_value))) #else #define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL) #endif static inline int _Py_EnterRecursivePy(PyThreadState *tstate) { return (tstate->py_recursion_remaining-- <= 0) && _Py_CheckRecursiveCallPy(tstate); } static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { tstate->py_recursion_remaining++; } /* Marker to specify tier 1 only instructions */ #define TIER_ONE_ONLY /* Marker to specify tier 2 only instructions */ #define TIER_TWO_ONLY /* Implementation of "macros" that modify the instruction pointer, * stack pointer, or frame pointer. * These need to treated differently by tier 1 and 2. * The Tier 1 version is here; Tier 2 is inlined in ceval.c. */ #define LOAD_IP(OFFSET) do { \ next_instr = frame->instr_ptr + (OFFSET); \ } while (0) /* There's no STORE_IP(), it's inlined by the code generator. */ #define LOAD_SP() \ stack_pointer = _PyFrame_GetStackPointer(frame); /* Tier-switching macros. */ #define GOTO_TIER_TWO() goto enter_tier_two; #define CURRENT_OPARG() (next_uop[-1].oparg) #define CURRENT_OPERAND() (next_uop[-1].operand)
generated by cgit v0.12 at 2025-08-11 20:40:35 (GMT)