/* NEON optimized code (C) COPYRIGHT 2009 Motorola * * Use of this source code is governed by a BSD-style license that can be * found in the LICENSE file. */ /* * Modifications done in-house at Motorola * * this is a clone of SkBitmapProcState_matrix.h * and has been tuned to work with the NEON unit. * * Still going back and forth between whether this approach * (clone the entire SkBitmapProcState_matrix.h file or * if I should put just the modified routines in here and * then use a construct like #define DONT_DO_THIS_FUNCTION or * something like that... * * This is for the RepeatX_RepeatY part of the world */ #if !defined(__ARM_HAVE_NEON) #error this file can be used only when the NEON unit is enabled #endif #include /* * This has been modified on the knowledge that (at the time) * we had the following macro definitions in the parent file * * #define MAKENAME(suffix) RepeatX_RepeatY ## suffix * #define TILEX_PROCF(fx, max) (((fx) & 0xFFFF) * ((max) + 1) >> 16) * #define TILEY_PROCF(fy, max) (((fy) & 0xFFFF) * ((max) + 1) >> 16) * #define TILEX_LOW_BITS(fx, max) ((((fx) & 0xFFFF) * ((max) + 1) >> 12) & 0xF) * #define TILEY_LOW_BITS(fy, max) ((((fy) & 0xFFFF) * ((max) + 1) >> 12) & 0xF) */ /* SkClampMax(val,max) -- bound to 0..max */ #define SCALE_NOFILTER_NAME MAKENAME(_nofilter_scale_neon) #define SCALE_FILTER_NAME MAKENAME(_filter_scale) #define AFFINE_NOFILTER_NAME MAKENAME(_nofilter_affine_neon) #define AFFINE_FILTER_NAME MAKENAME(_filter_affine) #define PERSP_NOFILTER_NAME MAKENAME(_nofilter_persp_neon) #define PERSP_FILTER_NAME MAKENAME(_filter_persp) #define PACK_FILTER_X_NAME MAKENAME(_pack_filter_x) #define PACK_FILTER_Y_NAME MAKENAME(_pack_filter_y) #ifndef PREAMBLE #define PREAMBLE(state) #define PREAMBLE_PARAM_X #define PREAMBLE_PARAM_Y #define PREAMBLE_ARG_X #define PREAMBLE_ARG_Y #endif static void SCALE_NOFILTER_NAME(const SkBitmapProcState& s, uint32_t xy[], int count, int x, int y) { SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | SkMatrix::kScale_Mask)) == 0); PREAMBLE(s); // we store y, x, x, x, x, x const unsigned maxX = s.fBitmap->width() - 1; SkFixed fx; { SkPoint pt; s.fInvProc(*s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, SkIntToScalar(y) + SK_ScalarHalf, &pt); fx = SkScalarToFixed(pt.fY); const unsigned maxY = s.fBitmap->height() - 1; *xy++ = TILEY_PROCF(fx, maxY); fx = SkScalarToFixed(pt.fX); } if (0 == maxX) { // all of the following X values must be 0 memset(xy, 0, count * sizeof(uint16_t)); return; } const SkFixed dx = s.fInvSx; #ifdef CHECK_FOR_DECAL // test if we don't need to apply the tile proc if ((unsigned)(fx >> 16) <= maxX && (unsigned)((fx + dx * (count - 1)) >> 16) <= maxX) { decal_nofilter_scale(xy, fx, dx, count); } else #endif { int i; #if defined(__ARM_HAVE_NEON) /* RBE: very much like done in decal_nofilter , * but some processing of the 'fx' information * TILEX_PROCF(fx, max) (((fx) & 0xFFFF) * ((max) + 1) >> 16) */ if (count >= 8) { /* SkFixed is 16.16 fixed point */ SkFixed dx2 = dx+dx; SkFixed dx4 = dx2+dx2; SkFixed dx8 = dx4+dx4; /* now build fx/fx+dx/fx+2dx/fx+3dx */ SkFixed fx1, fx2, fx3; int32x2_t lower, upper; int32x4_t lbase, hbase; int16_t *dst16 = (int16_t *)xy; fx1 = fx+dx; fx2 = fx1+dx; fx3 = fx2+dx; lbase = vdupq_n_s32(fx); lbase = vsetq_lane_s32(fx1, lbase, 1); lbase = vsetq_lane_s32(fx2, lbase, 2); lbase = vsetq_lane_s32(fx3, lbase, 3); hbase = vaddq_s32(lbase, vdupq_n_s32(dx4)); /* store & bump */ do { int32x4_t lout; int32x4_t hout; int16x8_t hi16; /* TILEX_PROCF(fx, max) (((fx)&0xFFFF)*((max)+1)>> 16) */ /* mask to low 16 [would like to use uzp tricks) */ lout = vandq_s32(lbase, vdupq_n_s32(0xffff)); hout = vandq_s32(hbase, vdupq_n_s32(0xffff)); /* bare multiplication, not SkFixedMul */ lout = vmulq_s32(lout, vdupq_n_s32(maxX+1)); hout = vmulq_s32(hout, vdupq_n_s32(maxX+1)); /* extraction, using uzp */ /* this is ok -- we want all hi(lout)s then all hi(hout)s */ asm ("vuzpq.16 %q0, %q1" : "+w" (lout), "+w" (hout)); hi16 = vreinterpretq_s16_s32(hout); vst1q_s16(dst16, hi16); /* bump our base on to the next */ lbase = vaddq_s32 (lbase, vdupq_n_s32(dx8)); hbase = vaddq_s32 (hbase, vdupq_n_s32(dx8)); dst16 += 8; count -= 8; fx += dx8; } while (count >= 8); xy = (uint32_t *) dst16; } #else /* simple, portable way of looking at 4 at a crack; * so gets some loop unrolling, but not full SIMD speed */ for (i = (count >> 2); i > 0; --i) { unsigned a, b; a = TILEX_PROCF(fx, maxX); fx += dx; b = TILEX_PROCF(fx, maxX); fx += dx; #ifdef SK_CPU_BENDIAN *xy++ = (a << 16) | b; #else *xy++ = (b << 16) | a; #endif a = TILEX_PROCF(fx, maxX); fx += dx; b = TILEX_PROCF(fx, maxX); fx += dx; #ifdef SK_CPU_BENDIAN *xy++ = (a << 16) | b; #else *xy++ = (b << 16) | a; #endif } /* loop doesn't adjust count */ count -= (count>>2); #endif uint16_t* xx = (uint16_t*)xy; for (i = count; i > 0; --i) { *xx++ = TILEX_PROCF(fx, maxX); fx += dx; } } } // note: we could special-case on a matrix which is skewed in X but not Y. // this would require a more general setup thatn SCALE does, but could use // SCALE's inner loop that only looks at dx static void AFFINE_NOFILTER_NAME(const SkBitmapProcState& s, uint32_t xy[], int count, int x, int y) { SkASSERT(s.fInvType & SkMatrix::kAffine_Mask); SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | SkMatrix::kScale_Mask | SkMatrix::kAffine_Mask)) == 0); PREAMBLE(s); SkPoint srcPt; s.fInvProc(*s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, SkIntToScalar(y) + SK_ScalarHalf, &srcPt); SkFixed fx = SkScalarToFixed(srcPt.fX); SkFixed fy = SkScalarToFixed(srcPt.fY); SkFixed dx = s.fInvSx; SkFixed dy = s.fInvKy; int maxX = s.fBitmap->width() - 1; int maxY = s.fBitmap->height() - 1; #if 1 int ocount = count; uint32_t *oxy = xy; SkFixed bfx = fx, bfy=fy, bdx=dx, bdy=dy; #endif #if defined(__ARM_HAVE_NEON) if (0) { extern void rbe(void); rbe(); } /* RBE: benchmarks show this eats up time; can we neonize it? */ /* RBE: very much like done in decal_nofilter , * but some processing of the 'fx' information * TILEX_PROCF(fx, max) (((fx) & 0xFFFF) * ((max) + 1) >> 16) */ if (count >= 4) { /* SkFixed is 16.16 fixed point */ SkFixed dx4 = dx*4; SkFixed dy4 = dy*4; /* now build fx/fx+dx/fx+2dx/fx+3dx */ int32x2_t lower, upper; int32x4_t xbase, ybase; int16_t *dst16 = (int16_t *)xy; /* synthesize 4x for both X and Y */ xbase = vdupq_n_s32(fx); xbase = vsetq_lane_s32(fx+dx, xbase, 1); xbase = vsetq_lane_s32(fx+dx+dx, xbase, 2); xbase = vsetq_lane_s32(fx+dx+dx+dx, xbase, 3); ybase = vdupq_n_s32(fy); ybase = vsetq_lane_s32(fy+dy, ybase, 1); ybase = vsetq_lane_s32(fy+dy+dy, ybase, 2); ybase = vsetq_lane_s32(fy+dy+dy+dy, ybase, 3); /* store & bump */ do { int32x4_t xout; int32x4_t yout; int16x8_t hi16; /* TILEX_PROCF(fx, max) (((fx)&0xFFFF)*((max)+1)>> 16) */ /* mask to low 16 [would like to use uzp tricks) */ xout = vandq_s32(xbase, vdupq_n_s32(0xffff)); yout = vandq_s32(ybase, vdupq_n_s32(0xffff)); /* bare multiplication, not SkFixedMul */ xout = vmulq_s32(xout, vdupq_n_s32(maxX+1)); yout = vmulq_s32(yout, vdupq_n_s32(maxY+1)); /* put hi16 from xout over low16 from yout */ yout = vsriq_n_s32(yout, xout, 16); /* and then yout has the interleaved upper 16's */ hi16 = vreinterpretq_s16_s32(yout); vst1q_s16(dst16, hi16); /* bump preserved base & on to the next */ xbase = vaddq_s32 (xbase, vdupq_n_s32(dx4)); ybase = vaddq_s32 (ybase, vdupq_n_s32(dy4)); dst16 += 8; /* 8 x16 aka 4x32 */ count -= 4; fx += dx4; fy += dy4; } while (count >= 4); xy = (uint32_t *) dst16; } #if 0 /* diagnostics... see whether we agree with the NEON code */ int bad = 0; uint32_t *myxy = oxy; int myi = (-1); SkFixed ofx = bfx, ofy= bfy, odx= bdx, ody= bdy; for (myi = ocount; myi > 0; --myi) { uint32_t val = (TILEY_PROCF(ofy, maxY) << 16) | TILEX_PROCF(ofx, maxX); if (val != *myxy++) { bad++; break; } ofx += odx; ofy += ody; } if (bad) { SkDebugf("repeat-nofilter-affine fails\n"); SkDebugf("count %d myi %d\n", ocount, myi); SkDebugf(" bfx %08x, bdx %08x, bfy %08x bdy %08x\n", bfx, bdx, bfy, bdy); SkDebugf("maxX %08x maxY %08x\n", maxX, maxY); } #endif #endif for (int i = count; i > 0; --i) { /* fx, fy, dx, dy are all 32 bit 16.16 fixed point */ /* (((fx) & 0xFFFF) * ((max) + 1) >> 16) */ *xy++ = (TILEY_PROCF(fy, maxY) << 16) | TILEX_PROCF(fx, maxX); fx += dx; fy += dy; } } static void PERSP_NOFILTER_NAME(const SkBitmapProcState& s, uint32_t* SK_RESTRICT xy, int count, int x, int y) { SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask); PREAMBLE(s); int maxX = s.fBitmap->width() - 1; int maxY = s.fBitmap->height() - 1; SkPerspIter iter(*s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, SkIntToScalar(y) + SK_ScalarHalf, count); while ((count = iter.next()) != 0) { const SkFixed* SK_RESTRICT srcXY = iter.getXY(); #if defined(__ARM_HAVE_NEON) /* RBE: */ /* TILEX_PROCF(fx, max) (((fx) & 0xFFFF) * ((max) + 1) >> 16) */ /* it's a little more complicated than what I did for the * clamp case -- where I could immediately snip to the top * 16 bits and do my min/max games there. * ... might only be able to get 4x unrolling here */ /* vld2 to get a set of 32x4's ... */ /* do the tile[xy]_procf operations */ /* which includes doing vuzp to get hi16's */ /* store it */ /* -- inner loop (other than vld2) can be had from above */ /* srcXY is a batch of 32 bit numbers X0,Y0,X1,Y1... * but we immediately discard the low 16 bits... * so what we're going to do is vld4, which will give us * xlo,xhi,ylo,yhi distribution and we can ignore the 'lo' * parts.... */ if (0) { extern void rbe(void); rbe(); } if (count >= 8) { int32_t *mysrc = (int32_t *) srcXY; int16_t *mydst = (int16_t *) xy; do { int32x4_t x, y, x2, y2; int16x8_t hi, hi2; /* read array of x,y,x,y,x,y */ /* vld2 does the de-interleaving for us */ /* isolate reg-bound scopes; gcc will minimize register * motion if possible; this ensures that we don't lose * a register across a debugging call because it happens * to be bound into a call-clobbered register */ { register int32x4_t q0 asm("q0"); register int32x4_t q1 asm("q1"); asm ("vld2.32 {q0-q1},[%2] /* x=%q0 y=%q1 */" : "=w" (q0), "=w" (q1) : "r" (mysrc) ); x = q0; y = q1; } /* offset == 256 bits == 32 bytes == 8 longs */ { register int32x4_t q2 asm("q2"); register int32x4_t q3 asm("q3"); asm ("vld2.32 {q2-q3},[%2] /* x=%q0 y=%q1 */" : "=w" (q2), "=w" (q3) : "r" (mysrc+8) ); x2 = q2; y2 = q3; } /* TILEX_PROCF(fx, max) (((fx)&0xFFFF)*((max)+1)>> 16) */ /* mask to low 16 [would like to use uzp tricks) */ /* bare multiplication, not SkFixedMul */ x = vandq_s32(x, vdupq_n_s32(0xffff)); x = vmulq_s32(x, vdupq_n_s32(maxX+1)); y = vandq_s32(y, vdupq_n_s32(0xffff)); y = vmulq_s32(y, vdupq_n_s32(maxY+1)); x2 = vandq_s32(x2, vdupq_n_s32(0xffff)); x2 = vmulq_s32(x2, vdupq_n_s32(maxX+1)); y2 = vandq_s32(y2, vdupq_n_s32(0xffff)); y2 = vmulq_s32(y2, vdupq_n_s32(maxY+1)); /* now collect interleaved high 16's */ /* (hi-x, hi-y)4 (hi-x2; hi-y2)4 */ /* extraction, using uzp, leaves hi16's in y */ y = vsriq_n_s32(y, x, 16); hi = vreinterpretq_s16_s32(y); vst1q_s16(mydst, hi); /* and likewise for the second 8 entries */ y2 = vsriq_n_s32(y2, x2, 16); hi2 = vreinterpretq_s16_s32(y2); vst1q_s16(mydst+8, hi2); /* XXX: gcc isn't interleaving these with the NEON ops * but i think that all the scoreboarding works out */ count -= 8; /* 8 iterations */ mysrc += 16; /* 16 longs */ mydst += 16; /* 16 shorts, aka 8 longs */ } while (count >= 8); /* get xy and srcXY fixed up */ srcXY = (const SkFixed *) mysrc; xy = (uint32_t *) mydst; } #endif while (--count >= 0) { *xy++ = (TILEY_PROCF(srcXY[1], maxY) << 16) | TILEX_PROCF(srcXY[0], maxX); srcXY += 2; } } } ////////////////////////////////////////////////////////////////////////////// static inline uint32_t PACK_FILTER_Y_NAME(SkFixed f, unsigned max, SkFixed one PREAMBLE_PARAM_Y) { unsigned i = TILEY_PROCF(f, max); i = (i << 4) | TILEY_LOW_BITS(f, max); return (i << 14) | (TILEY_PROCF((f + one), max)); } static inline uint32_t PACK_FILTER_X_NAME(SkFixed f, unsigned max, SkFixed one PREAMBLE_PARAM_X) { unsigned i = TILEX_PROCF(f, max); i = (i << 4) | TILEX_LOW_BITS(f, max); return (i << 14) | (TILEX_PROCF((f + one), max)); } static void SCALE_FILTER_NAME(const SkBitmapProcState& s, uint32_t xy[], int count, int x, int y) { SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | SkMatrix::kScale_Mask)) == 0); SkASSERT(s.fInvKy == 0); PREAMBLE(s); const unsigned maxX = s.fBitmap->width() - 1; const SkFixed one = s.fFilterOneX; const SkFixed dx = s.fInvSx; SkFixed fx; { SkPoint pt; s.fInvProc(*s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, SkIntToScalar(y) + SK_ScalarHalf, &pt); const SkFixed fy = SkScalarToFixed(pt.fY) - (s.fFilterOneY >> 1); const unsigned maxY = s.fBitmap->height() - 1; // compute our two Y values up front *xy++ = PACK_FILTER_Y_NAME(fy, maxY, s.fFilterOneY PREAMBLE_ARG_Y); // now initialize fx fx = SkScalarToFixed(pt.fX) - (one >> 1); } #ifdef CHECK_FOR_DECAL // test if we don't need to apply the tile proc if (dx > 0 && (unsigned)(fx >> 16) <= maxX && (unsigned)((fx + dx * (count - 1)) >> 16) < maxX) { decal_filter_scale(xy, fx, dx, count); } else #endif { do { *xy++ = PACK_FILTER_X_NAME(fx, maxX, one PREAMBLE_ARG_X); fx += dx; } while (--count != 0); } } static void AFFINE_FILTER_NAME(const SkBitmapProcState& s, uint32_t xy[], int count, int x, int y) { SkASSERT(s.fInvType & SkMatrix::kAffine_Mask); SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | SkMatrix::kScale_Mask | SkMatrix::kAffine_Mask)) == 0); PREAMBLE(s); SkPoint srcPt; s.fInvProc(*s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, SkIntToScalar(y) + SK_ScalarHalf, &srcPt); SkFixed oneX = s.fFilterOneX; SkFixed oneY = s.fFilterOneY; SkFixed fx = SkScalarToFixed(srcPt.fX) - (oneX >> 1); SkFixed fy = SkScalarToFixed(srcPt.fY) - (oneY >> 1); SkFixed dx = s.fInvSx; SkFixed dy = s.fInvKy; unsigned maxX = s.fBitmap->width() - 1; unsigned maxY = s.fBitmap->height() - 1; do { *xy++ = PACK_FILTER_Y_NAME(fy, maxY, oneY PREAMBLE_ARG_Y); fy += dy; *xy++ = PACK_FILTER_X_NAME(fx, maxX, oneX PREAMBLE_ARG_X); fx += dx; } while (--count != 0); } static void PERSP_FILTER_NAME(const SkBitmapProcState& s, uint32_t* SK_RESTRICT xy, int count, int x, int y) { SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask); extern void rbe(void); PREAMBLE(s); unsigned maxX = s.fBitmap->width() - 1; unsigned maxY = s.fBitmap->height() - 1; SkFixed oneX = s.fFilterOneX; SkFixed oneY = s.fFilterOneY; SkPerspIter iter(*s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, SkIntToScalar(y) + SK_ScalarHalf, count); while ((count = iter.next()) != 0) { const SkFixed* SK_RESTRICT srcXY = iter.getXY(); do { *xy++ = PACK_FILTER_Y_NAME(srcXY[1] - (oneY >> 1), maxY, oneY PREAMBLE_ARG_Y); *xy++ = PACK_FILTER_X_NAME(srcXY[0] - (oneX >> 1), maxX, oneX PREAMBLE_ARG_X); srcXY += 2; } while (--count != 0); } } static SkBitmapProcState::MatrixProc MAKENAME(_Procs)[] = { SCALE_NOFILTER_NAME, SCALE_FILTER_NAME, AFFINE_NOFILTER_NAME, AFFINE_FILTER_NAME, PERSP_NOFILTER_NAME, PERSP_FILTER_NAME }; #undef MAKENAME #undef TILEX_PROCF #undef TILEY_PROCF #ifdef CHECK_FOR_DECAL #undef CHECK_FOR_DECAL #endif #undef SCALE_NOFILTER_NAME #undef SCALE_FILTER_NAME #undef AFFINE_NOFILTER_NAME #undef AFFINE_FILTER_NAME #undef PERSP_NOFILTER_NAME #undef PERSP_FILTER_NAME #undef PREAMBLE #undef PREAMBLE_PARAM_X #undef PREAMBLE_PARAM_Y #undef PREAMBLE_ARG_X #undef PREAMBLE_ARG_Y #undef TILEX_LOW_BITS #undef TILEY_LOW_BITS