From 0700471c72bca247109af0165d09173ce9c107c8 Mon Sep 17 00:00:00 2001 From: "jzern@chromium.org" Date: Sun, 24 Mar 2013 23:13:30 +0000 Subject: libwebp: update snapshot to v0.3.0-rc6 adds metadata (XMP/EXIF), ICC profile and animation support to the format. BUG=https://bugs.webkit.org/show_bug.cgi?id=112761 TEST=build,webkit layout TBR=cpu Review URL: https://chromiumcodereview.appspot.com/12942006 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@190302 0039d316-1c4b-4281-b951-d872f2087c98 --- third_party/libwebp/dsp/dec.c | 15 +- third_party/libwebp/dsp/dec_neon.c | 88 +++- third_party/libwebp/dsp/dec_sse2.c | 37 +- third_party/libwebp/dsp/dsp.h | 17 +- third_party/libwebp/dsp/enc.c | 117 +++--- third_party/libwebp/dsp/enc_neon.c | 661 ++++++++++++++++++++++++++++++ third_party/libwebp/dsp/enc_sse2.c | 321 ++++++++++----- third_party/libwebp/dsp/lossless.c | 370 +++++++++++++---- third_party/libwebp/dsp/lossless.h | 16 +- third_party/libwebp/dsp/upsampling.c | 28 +- third_party/libwebp/dsp/upsampling_neon.c | 292 +++++++++++++ third_party/libwebp/dsp/upsampling_sse2.c | 38 +- third_party/libwebp/dsp/yuv.c | 23 +- third_party/libwebp/dsp/yuv.h | 219 ++++++++-- 14 files changed, 1885 insertions(+), 357 deletions(-) create mode 100644 third_party/libwebp/dsp/enc_neon.c create mode 100644 third_party/libwebp/dsp/upsampling_neon.c (limited to 'third_party/libwebp/dsp') diff --git a/third_party/libwebp/dsp/dec.c b/third_party/libwebp/dsp/dec.c index 9ae7b6f..758c6a5 100644 --- a/third_party/libwebp/dsp/dec.c +++ b/third_party/libwebp/dsp/dec.c @@ -426,11 +426,16 @@ static void HE8uv(uint8_t *dst) { // horizontal } // helper for chroma-DC predictions -static WEBP_INLINE void Put8x8uv(uint64_t v, uint8_t* dst) { +static WEBP_INLINE void Put8x8uv(uint8_t value, uint8_t* dst) { int j; +#ifndef WEBP_REFERENCE_IMPLEMENTATION + const uint64_t v = (uint64_t)value * 0x0101010101010101ULL; for (j = 0; j < 8; ++j) { *(uint64_t*)(dst + j * BPS) = v; } +#else + for (j = 0; j < 8; ++j) memset(dst + j * BPS, value, 8); +#endif } static void DC8uv(uint8_t *dst) { // DC @@ -439,7 +444,7 @@ static void DC8uv(uint8_t *dst) { // DC for (i = 0; i < 8; ++i) { dc0 += dst[i - BPS] + dst[-1 + i * BPS]; } - Put8x8uv((uint64_t)((dc0 >> 4) * 0x0101010101010101ULL), dst); + Put8x8uv(dc0 >> 4, dst); } static void DC8uvNoLeft(uint8_t *dst) { // DC with no left samples @@ -448,7 +453,7 @@ static void DC8uvNoLeft(uint8_t *dst) { // DC with no left samples for (i = 0; i < 8; ++i) { dc0 += dst[i - BPS]; } - Put8x8uv((uint64_t)((dc0 >> 3) * 0x0101010101010101ULL), dst); + Put8x8uv(dc0 >> 3, dst); } static void DC8uvNoTop(uint8_t *dst) { // DC with no top samples @@ -457,11 +462,11 @@ static void DC8uvNoTop(uint8_t *dst) { // DC with no top samples for (i = 0; i < 8; ++i) { dc0 += dst[-1 + i * BPS]; } - Put8x8uv((uint64_t)((dc0 >> 3) * 0x0101010101010101ULL), dst); + Put8x8uv(dc0 >> 3, dst); } static void DC8uvNoTopLeft(uint8_t *dst) { // DC with nothing - Put8x8uv(0x8080808080808080ULL, dst); + Put8x8uv(0x80, dst); } //------------------------------------------------------------------------------ diff --git a/third_party/libwebp/dsp/dec_neon.c b/third_party/libwebp/dsp/dec_neon.c index ec824b7..5d7cff1 100644 --- a/third_party/libwebp/dsp/dec_neon.c +++ b/third_party/libwebp/dsp/dec_neon.c @@ -12,14 +12,14 @@ #include "./dsp.h" -#if defined(WEBP_USE_NEON) - -#include "../dec/vp8i.h" - #if defined(__cplusplus) || defined(c_plusplus) extern "C" { #endif +#if defined(WEBP_USE_NEON) + +#include "../dec/vp8i.h" + #define QRegs "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", \ "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" @@ -155,6 +155,9 @@ static void SimpleHFilter16iNEON(uint8_t* p, int stride, int thresh) { } } +//----------------------------------------------------------------------------- +// Inverse transforms (Paragraph 14.4) + static void TransformOneNEON(const int16_t *in, uint8_t *dst) { const int kBPS = BPS; const int16_t constants[] = {20091, 17734, 0, 0}; @@ -311,19 +314,92 @@ static void TransformTwoNEON(const int16_t* in, uint8_t* dst, int do_two) { } } +static void TransformWHT(const int16_t* in, int16_t* out) { + const int kStep = 32; // The store is only incrementing the pointer as if we + // had stored a single byte. + __asm__ volatile ( + // part 1 + // load data into q0, q1 + "vld1.16 {q0, q1}, [%[in]] \n" + + "vaddl.s16 q2, d0, d3 \n" // a0 = in[0] + in[12] + "vaddl.s16 q3, d1, d2 \n" // a1 = in[4] + in[8] + "vsubl.s16 q4, d1, d2 \n" // a2 = in[4] - in[8] + "vsubl.s16 q5, d0, d3 \n" // a3 = in[0] - in[12] + + "vadd.s32 q0, q2, q3 \n" // tmp[0] = a0 + a1 + "vsub.s32 q2, q2, q3 \n" // tmp[8] = a0 - a1 + "vadd.s32 q1, q5, q4 \n" // tmp[4] = a3 + a2 + "vsub.s32 q3, q5, q4 \n" // tmp[12] = a3 - a2 + + // Transpose + // q0 = tmp[0, 4, 8, 12], q1 = tmp[2, 6, 10, 14] + // q2 = tmp[1, 5, 9, 13], q3 = tmp[3, 7, 11, 15] + "vswp d1, d4 \n" // vtrn.64 q0, q2 + "vswp d3, d6 \n" // vtrn.64 q1, q3 + "vtrn.32 q0, q1 \n" + "vtrn.32 q2, q3 \n" + + "vmov.s32 q4, #3 \n" // dc = 3 + "vadd.s32 q0, q0, q4 \n" // dc = tmp[0] + 3 + "vadd.s32 q6, q0, q3 \n" // a0 = dc + tmp[3] + "vadd.s32 q7, q1, q2 \n" // a1 = tmp[1] + tmp[2] + "vsub.s32 q8, q1, q2 \n" // a2 = tmp[1] - tmp[2] + "vsub.s32 q9, q0, q3 \n" // a3 = dc - tmp[3] + + "vadd.s32 q0, q6, q7 \n" + "vshrn.s32 d0, q0, #3 \n" // (a0 + a1) >> 3 + "vadd.s32 q1, q9, q8 \n" + "vshrn.s32 d1, q1, #3 \n" // (a3 + a2) >> 3 + "vsub.s32 q2, q6, q7 \n" + "vshrn.s32 d2, q2, #3 \n" // (a0 - a1) >> 3 + "vsub.s32 q3, q9, q8 \n" + "vshrn.s32 d3, q3, #3 \n" // (a3 - a2) >> 3 + + // set the results to output + "vst1.16 d0[0], [%[out]], %[kStep] \n" + "vst1.16 d1[0], [%[out]], %[kStep] \n" + "vst1.16 d2[0], [%[out]], %[kStep] \n" + "vst1.16 d3[0], [%[out]], %[kStep] \n" + "vst1.16 d0[1], [%[out]], %[kStep] \n" + "vst1.16 d1[1], [%[out]], %[kStep] \n" + "vst1.16 d2[1], [%[out]], %[kStep] \n" + "vst1.16 d3[1], [%[out]], %[kStep] \n" + "vst1.16 d0[2], [%[out]], %[kStep] \n" + "vst1.16 d1[2], [%[out]], %[kStep] \n" + "vst1.16 d2[2], [%[out]], %[kStep] \n" + "vst1.16 d3[2], [%[out]], %[kStep] \n" + "vst1.16 d0[3], [%[out]], %[kStep] \n" + "vst1.16 d1[3], [%[out]], %[kStep] \n" + "vst1.16 d2[3], [%[out]], %[kStep] \n" + "vst1.16 d3[3], [%[out]], %[kStep] \n" + + : [out] "+r"(out) // modified registers + : [in] "r"(in), [kStep] "r"(kStep) // constants + : "memory", "q0", "q1", "q2", "q3", "q4", + "q5", "q6", "q7", "q8", "q9" // clobbered + ); +} + +#endif // WEBP_USE_NEON + +//------------------------------------------------------------------------------ +// Entry point + extern void VP8DspInitNEON(void); void VP8DspInitNEON(void) { +#if defined(WEBP_USE_NEON) VP8Transform = TransformTwoNEON; + VP8TransformWHT = TransformWHT; VP8SimpleVFilter16 = SimpleVFilter16NEON; VP8SimpleHFilter16 = SimpleHFilter16NEON; VP8SimpleVFilter16i = SimpleVFilter16iNEON; VP8SimpleHFilter16i = SimpleHFilter16iNEON; +#endif // WEBP_USE_NEON } #if defined(__cplusplus) || defined(c_plusplus) } // extern "C" #endif - -#endif // WEBP_USE_NEON diff --git a/third_party/libwebp/dsp/dec_sse2.c b/third_party/libwebp/dsp/dec_sse2.c index 472b68e..1cac1b8 100644 --- a/third_party/libwebp/dsp/dec_sse2.c +++ b/third_party/libwebp/dsp/dec_sse2.c @@ -12,15 +12,15 @@ #include "./dsp.h" +#if defined(__cplusplus) || defined(c_plusplus) +extern "C" { +#endif + #if defined(WEBP_USE_SSE2) #include #include "../dec/vp8i.h" -#if defined(__cplusplus) || defined(c_plusplus) -extern "C" { -#endif - //------------------------------------------------------------------------------ // Transforms (Paragraph 14.4) @@ -194,7 +194,7 @@ static void TransformSSE2(const int16_t* in, uint8_t* dst, int do_two) { // Add inverse transform to 'dst' and store. { - const __m128i zero = _mm_set1_epi16(0); + const __m128i zero = _mm_setzero_si128(); // Load the reference(s). __m128i dst0, dst1, dst2, dst3; if (do_two) { @@ -278,14 +278,14 @@ static void TransformSSE2(const int16_t* in, uint8_t* dst, int do_two) { #define GET_NOTHEV(p1, p0, q0, q1, hev_thresh, not_hev) { \ const __m128i zero = _mm_setzero_si128(); \ - const __m128i t1 = MM_ABS(p1, p0); \ - const __m128i t2 = MM_ABS(q1, q0); \ + const __m128i t_1 = MM_ABS(p1, p0); \ + const __m128i t_2 = MM_ABS(q1, q0); \ \ const __m128i h = _mm_set1_epi8(hev_thresh); \ - const __m128i t3 = _mm_subs_epu8(t1, h); /* abs(p1 - p0) - hev_tresh */ \ - const __m128i t4 = _mm_subs_epu8(t2, h); /* abs(q1 - q0) - hev_tresh */ \ + const __m128i t_3 = _mm_subs_epu8(t_1, h); /* abs(p1 - p0) - hev_tresh */ \ + const __m128i t_4 = _mm_subs_epu8(t_2, h); /* abs(q1 - q0) - hev_tresh */ \ \ - not_hev = _mm_or_si128(t3, t4); \ + not_hev = _mm_or_si128(t_3, t_4); \ not_hev = _mm_cmpeq_epi8(not_hev, zero); /* not_hev <= t1 && not_hev <= t2 */\ } @@ -314,13 +314,13 @@ static void TransformSSE2(const int16_t* in, uint8_t* dst, int do_two) { // Updates values of 2 pixels at MB edge during complex filtering. // Update operations: -// q = q - a and p = p + a; where a = [(a_hi >> 7), (a_lo >> 7)] +// q = q - delta and p = p + delta; where delta = [(a_hi >> 7), (a_lo >> 7)] #define UPDATE_2PIXELS(pi, qi, a_lo, a_hi) { \ const __m128i a_lo7 = _mm_srai_epi16(a_lo, 7); \ const __m128i a_hi7 = _mm_srai_epi16(a_hi, 7); \ - const __m128i a = _mm_packs_epi16(a_lo7, a_hi7); \ - pi = _mm_adds_epi8(pi, a); \ - qi = _mm_subs_epi8(qi, a); \ + const __m128i delta = _mm_packs_epi16(a_lo7, a_hi7); \ + pi = _mm_adds_epi8(pi, delta); \ + qi = _mm_subs_epi8(qi, delta); \ } static void NeedsFilter(const __m128i* p1, const __m128i* p0, const __m128i* q0, @@ -876,9 +876,15 @@ static void HFilter8iSSE2(uint8_t* u, uint8_t* v, int stride, Store16x4(u, v, stride, &p1, &p0, &q0, &q1); } +#endif // WEBP_USE_SSE2 + +//------------------------------------------------------------------------------ +// Entry point + extern void VP8DspInitSSE2(void); void VP8DspInitSSE2(void) { +#if defined(WEBP_USE_SSE2) VP8Transform = TransformSSE2; VP8VFilter16 = VFilter16SSE2; @@ -894,10 +900,9 @@ void VP8DspInitSSE2(void) { VP8SimpleHFilter16 = SimpleHFilter16SSE2; VP8SimpleVFilter16i = SimpleVFilter16iSSE2; VP8SimpleHFilter16i = SimpleHFilter16iSSE2; +#endif // WEBP_USE_SSE2 } #if defined(__cplusplus) || defined(c_plusplus) } // extern "C" #endif - -#endif // WEBP_USE_SSE2 diff --git a/third_party/libwebp/dsp/dsp.h b/third_party/libwebp/dsp/dsp.h index 24d0c18..933df60 100644 --- a/third_party/libwebp/dsp/dsp.h +++ b/third_party/libwebp/dsp/dsp.h @@ -49,8 +49,6 @@ extern VP8CPUInfo VP8GetCPUInfo; //------------------------------------------------------------------------------ // Encoding -int VP8GetAlpha(const int histo[]); - // Transforms // VP8Idct: Does one of two inverse transforms. If do_two is set, the transforms // will be done for (ref, in, dst) and (ref + 4, in + 16, dst + 4). @@ -85,10 +83,11 @@ typedef int (*VP8QuantizeBlock)(int16_t in[16], int16_t out[16], int n, const struct VP8Matrix* const mtx); extern VP8QuantizeBlock VP8EncQuantizeBlock; -// Compute susceptibility based on DCT-coeff histograms: -// the higher, the "easier" the macroblock is to compress. -typedef int (*VP8CHisto)(const uint8_t* ref, const uint8_t* pred, - int start_block, int end_block); +// Collect histogram for susceptibility calculation and accumulate in histo[]. +struct VP8Histogram; +typedef void (*VP8CHisto)(const uint8_t* ref, const uint8_t* pred, + int start_block, int end_block, + struct VP8Histogram* const histo); extern const int VP8DspScan[16 + 4 + 4]; extern VP8CHisto VP8CollectHistogram; @@ -104,7 +103,7 @@ extern VP8DecIdct2 VP8Transform; extern VP8DecIdct VP8TransformUV; extern VP8DecIdct VP8TransformDC; extern VP8DecIdct VP8TransformDCUV; -extern void (*VP8TransformWHT)(const int16_t* in, int16_t* out); +extern VP8WHT VP8TransformWHT; // *dst is the destination block, with stride BPS. Boundary samples are // assumed accessible when needed. @@ -159,6 +158,9 @@ extern WebPUpsampleLinePairFunc WebPUpsamplers[/* MODE_LAST */]; // Initializes SSE2 version of the fancy upsamplers. void WebPInitUpsamplersSSE2(void); +// NEON version +void WebPInitUpsamplersNEON(void); + #endif // FANCY_UPSAMPLING // Point-sampling methods. @@ -200,6 +202,7 @@ extern void (*WebPApplyAlphaMultiply4444)( void WebPInitPremultiply(void); void WebPInitPremultiplySSE2(void); // should not be called directly. +void WebPInitPremultiplyNEON(void); //------------------------------------------------------------------------------ diff --git a/third_party/libwebp/dsp/enc.c b/third_party/libwebp/dsp/enc.c index 0223456..ae2c830 100644 --- a/third_party/libwebp/dsp/enc.c +++ b/third_party/libwebp/dsp/enc.c @@ -17,31 +17,18 @@ extern "C" { #endif -//------------------------------------------------------------------------------ -// Compute susceptibility based on DCT-coeff histograms: -// the higher, the "easier" the macroblock is to compress. - -static int ClipAlpha(int alpha) { - return alpha < 0 ? 0 : alpha > 255 ? 255 : alpha; +static WEBP_INLINE uint8_t clip_8b(int v) { + return (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255; } -int VP8GetAlpha(const int histo[MAX_COEFF_THRESH + 1]) { - int num = 0, den = 0, val = 0; - int k; - int alpha; - // note: changing this loop to avoid the numerous "k + 1" slows things down. - for (k = 0; k < MAX_COEFF_THRESH; ++k) { - if (histo[k + 1]) { - val += histo[k + 1]; - num += val * (k + 1); - den += (k + 1) * (k + 1); - } - } - // we scale the value to a usable [0..255] range - alpha = den ? 10 * num / den - 5 : 0; - return ClipAlpha(alpha); +static WEBP_INLINE int clip_max(int v, int max) { + return (v > max) ? max : v; } +//------------------------------------------------------------------------------ +// Compute susceptibility based on DCT-coeff histograms: +// the higher, the "easier" the macroblock is to compress. + const int VP8DspScan[16 + 4 + 4] = { // Luma 0 + 0 * BPS, 4 + 0 * BPS, 8 + 0 * BPS, 12 + 0 * BPS, @@ -53,27 +40,23 @@ const int VP8DspScan[16 + 4 + 4] = { 8 + 0 * BPS, 12 + 0 * BPS, 8 + 4 * BPS, 12 + 4 * BPS // V }; -static int CollectHistogram(const uint8_t* ref, const uint8_t* pred, - int start_block, int end_block) { - int histo[MAX_COEFF_THRESH + 1] = { 0 }; - int16_t out[16]; - int j, k; +static void CollectHistogram(const uint8_t* ref, const uint8_t* pred, + int start_block, int end_block, + VP8Histogram* const histo) { + int j; for (j = start_block; j < end_block; ++j) { - VP8FTransform(ref + VP8DspScan[j], pred + VP8DspScan[j], out); + int k; + int16_t out[16]; - // Convert coefficients to bin (within out[]). - for (k = 0; k < 16; ++k) { - const int v = abs(out[k]) >> 2; - out[k] = (v > MAX_COEFF_THRESH) ? MAX_COEFF_THRESH : v; - } + VP8FTransform(ref + VP8DspScan[j], pred + VP8DspScan[j], out); - // Use bin to update histogram. + // Convert coefficients to bin. for (k = 0; k < 16; ++k) { - histo[out[k]]++; + const int v = abs(out[k]) >> 3; // TODO(skal): add rounding? + const int clipped_value = clip_max(v, MAX_COEFF_THRESH); + histo->distribution[clipped_value]++; } } - - return VP8GetAlpha(histo); } //------------------------------------------------------------------------------ @@ -89,15 +72,12 @@ static void InitTables(void) { if (!tables_ok) { int i; for (i = -255; i <= 255 + 255; ++i) { - clip1[255 + i] = (i < 0) ? 0 : (i > 255) ? 255 : i; + clip1[255 + i] = clip_8b(i); } tables_ok = 1; } } -static WEBP_INLINE uint8_t clip_8b(int v) { - return (!(v & ~0xff)) ? v : v < 0 ? 0 : 255; -} //------------------------------------------------------------------------------ // Transforms (Paragraph 14.4) @@ -154,25 +134,25 @@ static void FTransform(const uint8_t* src, const uint8_t* ref, int16_t* out) { int i; int tmp[16]; for (i = 0; i < 4; ++i, src += BPS, ref += BPS) { - const int d0 = src[0] - ref[0]; + const int d0 = src[0] - ref[0]; // 9bit dynamic range ([-255,255]) const int d1 = src[1] - ref[1]; const int d2 = src[2] - ref[2]; const int d3 = src[3] - ref[3]; - const int a0 = (d0 + d3) << 3; - const int a1 = (d1 + d2) << 3; - const int a2 = (d1 - d2) << 3; - const int a3 = (d0 - d3) << 3; - tmp[0 + i * 4] = (a0 + a1); - tmp[1 + i * 4] = (a2 * 2217 + a3 * 5352 + 14500) >> 12; - tmp[2 + i * 4] = (a0 - a1); - tmp[3 + i * 4] = (a3 * 2217 - a2 * 5352 + 7500) >> 12; + const int a0 = (d0 + d3); // 10b [-510,510] + const int a1 = (d1 + d2); + const int a2 = (d1 - d2); + const int a3 = (d0 - d3); + tmp[0 + i * 4] = (a0 + a1) << 3; // 14b [-8160,8160] + tmp[1 + i * 4] = (a2 * 2217 + a3 * 5352 + 1812) >> 9; // [-7536,7542] + tmp[2 + i * 4] = (a0 - a1) << 3; + tmp[3 + i * 4] = (a3 * 2217 - a2 * 5352 + 937) >> 9; } for (i = 0; i < 4; ++i) { - const int a0 = (tmp[0 + i] + tmp[12 + i]); + const int a0 = (tmp[0 + i] + tmp[12 + i]); // 15b const int a1 = (tmp[4 + i] + tmp[ 8 + i]); const int a2 = (tmp[4 + i] - tmp[ 8 + i]); const int a3 = (tmp[0 + i] - tmp[12 + i]); - out[0 + i] = (a0 + a1 + 7) >> 4; + out[0 + i] = (a0 + a1 + 7) >> 4; // 12b out[4 + i] = ((a2 * 2217 + a3 * 5352 + 12000) >> 16) + (a3 != 0); out[8 + i] = (a0 - a1 + 7) >> 4; out[12+ i] = ((a3 * 2217 - a2 * 5352 + 51000) >> 16); @@ -589,30 +569,30 @@ static int TTransform(const uint8_t* in, const uint16_t* w) { int i; // horizontal pass for (i = 0; i < 4; ++i, in += BPS) { - const int a0 = (in[0] + in[2]) << 2; - const int a1 = (in[1] + in[3]) << 2; - const int a2 = (in[1] - in[3]) << 2; - const int a3 = (in[0] - in[2]) << 2; - tmp[0 + i * 4] = a0 + a1 + (a0 != 0); + const int a0 = in[0] + in[2]; + const int a1 = in[1] + in[3]; + const int a2 = in[1] - in[3]; + const int a3 = in[0] - in[2]; + tmp[0 + i * 4] = a0 + a1; tmp[1 + i * 4] = a3 + a2; tmp[2 + i * 4] = a3 - a2; tmp[3 + i * 4] = a0 - a1; } // vertical pass for (i = 0; i < 4; ++i, ++w) { - const int a0 = (tmp[0 + i] + tmp[8 + i]); - const int a1 = (tmp[4 + i] + tmp[12+ i]); - const int a2 = (tmp[4 + i] - tmp[12+ i]); - const int a3 = (tmp[0 + i] - tmp[8 + i]); + const int a0 = tmp[0 + i] + tmp[8 + i]; + const int a1 = tmp[4 + i] + tmp[12+ i]; + const int a2 = tmp[4 + i] - tmp[12+ i]; + const int a3 = tmp[0 + i] - tmp[8 + i]; const int b0 = a0 + a1; const int b1 = a3 + a2; const int b2 = a3 - a2; const int b3 = a0 - a1; - // abs((b + (b<0) + 3) >> 3) = (abs(b) + 3) >> 3 - sum += w[ 0] * ((abs(b0) + 3) >> 3); - sum += w[ 4] * ((abs(b1) + 3) >> 3); - sum += w[ 8] * ((abs(b2) + 3) >> 3); - sum += w[12] * ((abs(b3) + 3) >> 3); + + sum += w[ 0] * abs(b0); + sum += w[ 4] * abs(b1); + sum += w[ 8] * abs(b2); + sum += w[12] * abs(b3); } return sum; } @@ -621,7 +601,7 @@ static int Disto4x4(const uint8_t* const a, const uint8_t* const b, const uint16_t* const w) { const int sum1 = TTransform(a, w); const int sum2 = TTransform(b, w); - return (abs(sum2 - sum1) + 8) >> 4; + return abs(sum2 - sum1) >> 5; } static int Disto16x16(const uint8_t* const a, const uint8_t* const b, @@ -706,6 +686,7 @@ VP8QuantizeBlock VP8EncQuantizeBlock; VP8BlockCopy VP8Copy4x4; extern void VP8EncDspInitSSE2(void); +extern void VP8EncDspInitNEON(void); void VP8EncDspInit(void) { InitTables(); @@ -734,6 +715,10 @@ void VP8EncDspInit(void) { if (VP8GetCPUInfo(kSSE2)) { VP8EncDspInitSSE2(); } +#elif defined(WEBP_USE_NEON) + if (VP8GetCPUInfo(kNEON)) { + VP8EncDspInitNEON(); + } #endif } } diff --git a/third_party/libwebp/dsp/enc_neon.c b/third_party/libwebp/dsp/enc_neon.c new file mode 100644 index 0000000..b5a1fba --- /dev/null +++ b/third_party/libwebp/dsp/enc_neon.c @@ -0,0 +1,661 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// This code is licensed under the same terms as WebM: +// Software License Agreement: http://www.webmproject.org/license/software/ +// Additional IP Rights Grant: http://www.webmproject.org/license/additional/ +// ----------------------------------------------------------------------------- +// +// ARM NEON version of speed-critical encoding functions. +// +// adapted from libvpx (http://www.webmproject.org/code/) + +#include "./dsp.h" + +#if defined(__cplusplus) || defined(c_plusplus) +extern "C" { +#endif + +#if defined(WEBP_USE_NEON) + +#include "../enc/vp8enci.h" + +//------------------------------------------------------------------------------ +// Transforms (Paragraph 14.4) + +// Inverse transform. +// This code is pretty much the same as TransformOneNEON in the decoder, except +// for subtraction to *ref. See the comments there for algorithmic explanations. +static void ITransformOne(const uint8_t* ref, + const int16_t* in, uint8_t* dst) { + const int kBPS = BPS; + const int16_t kC1C2[] = { 20091, 17734, 0, 0 }; // kC1 / (kC2 >> 1) / 0 / 0 + + __asm__ volatile ( + "vld1.16 {q1, q2}, [%[in]] \n" + "vld1.16 {d0}, [%[kC1C2]] \n" + + // d2: in[0] + // d3: in[8] + // d4: in[4] + // d5: in[12] + "vswp d3, d4 \n" + + // q8 = {in[4], in[12]} * kC1 * 2 >> 16 + // q9 = {in[4], in[12]} * kC2 >> 16 + "vqdmulh.s16 q8, q2, d0[0] \n" + "vqdmulh.s16 q9, q2, d0[1] \n" + + // d22 = a = in[0] + in[8] + // d23 = b = in[0] - in[8] + "vqadd.s16 d22, d2, d3 \n" + "vqsub.s16 d23, d2, d3 \n" + + // q8 = in[4]/[12] * kC1 >> 16 + "vshr.s16 q8, q8, #1 \n" + + // Add {in[4], in[12]} back after the multiplication. + "vqadd.s16 q8, q2, q8 \n" + + // d20 = c = in[4]*kC2 - in[12]*kC1 + // d21 = d = in[4]*kC1 + in[12]*kC2 + "vqsub.s16 d20, d18, d17 \n" + "vqadd.s16 d21, d19, d16 \n" + + // d2 = tmp[0] = a + d + // d3 = tmp[1] = b + c + // d4 = tmp[2] = b - c + // d5 = tmp[3] = a - d + "vqadd.s16 d2, d22, d21 \n" + "vqadd.s16 d3, d23, d20 \n" + "vqsub.s16 d4, d23, d20 \n" + "vqsub.s16 d5, d22, d21 \n" + + "vzip.16 q1, q2 \n" + "vzip.16 q1, q2 \n" + + "vswp d3, d4 \n" + + // q8 = {tmp[4], tmp[12]} * kC1 * 2 >> 16 + // q9 = {tmp[4], tmp[12]} * kC2 >> 16 + "vqdmulh.s16 q8, q2, d0[0] \n" + "vqdmulh.s16 q9, q2, d0[1] \n" + + // d22 = a = tmp[0] + tmp[8] + // d23 = b = tmp[0] - tmp[8] + "vqadd.s16 d22, d2, d3 \n" + "vqsub.s16 d23, d2, d3 \n" + + "vshr.s16 q8, q8, #1 \n" + "vqadd.s16 q8, q2, q8 \n" + + // d20 = c = in[4]*kC2 - in[12]*kC1 + // d21 = d = in[4]*kC1 + in[12]*kC2 + "vqsub.s16 d20, d18, d17 \n" + "vqadd.s16 d21, d19, d16 \n" + + // d2 = tmp[0] = a + d + // d3 = tmp[1] = b + c + // d4 = tmp[2] = b - c + // d5 = tmp[3] = a - d + "vqadd.s16 d2, d22, d21 \n" + "vqadd.s16 d3, d23, d20 \n" + "vqsub.s16 d4, d23, d20 \n" + "vqsub.s16 d5, d22, d21 \n" + + "vld1.32 d6[0], [%[ref]], %[kBPS] \n" + "vld1.32 d6[1], [%[ref]], %[kBPS] \n" + "vld1.32 d7[0], [%[ref]], %[kBPS] \n" + "vld1.32 d7[1], [%[ref]], %[kBPS] \n" + + "sub %[ref], %[ref], %[kBPS], lsl #2 \n" + + // (val) + 4 >> 3 + "vrshr.s16 d2, d2, #3 \n" + "vrshr.s16 d3, d3, #3 \n" + "vrshr.s16 d4, d4, #3 \n" + "vrshr.s16 d5, d5, #3 \n" + + "vzip.16 q1, q2 \n" + "vzip.16 q1, q2 \n" + + // Must accumulate before saturating + "vmovl.u8 q8, d6 \n" + "vmovl.u8 q9, d7 \n" + + "vqadd.s16 q1, q1, q8 \n" + "vqadd.s16 q2, q2, q9 \n" + + "vqmovun.s16 d0, q1 \n" + "vqmovun.s16 d1, q2 \n" + + "vst1.32 d0[0], [%[dst]], %[kBPS] \n" + "vst1.32 d0[1], [%[dst]], %[kBPS] \n" + "vst1.32 d1[0], [%[dst]], %[kBPS] \n" + "vst1.32 d1[1], [%[dst]] \n" + + : [in] "+r"(in), [dst] "+r"(dst) // modified registers + : [kBPS] "r"(kBPS), [kC1C2] "r"(kC1C2), [ref] "r"(ref) // constants + : "memory", "q0", "q1", "q2", "q8", "q9", "q10", "q11" // clobbered + ); +} + +static void ITransform(const uint8_t* ref, + const int16_t* in, uint8_t* dst, int do_two) { + ITransformOne(ref, in, dst); + if (do_two) { + ITransformOne(ref + 4, in + 16, dst + 4); + } +} + +// Same code as dec_neon.c +static void ITransformWHT(const int16_t* in, int16_t* out) { + const int kStep = 32; // The store is only incrementing the pointer as if we + // had stored a single byte. + __asm__ volatile ( + // part 1 + // load data into q0, q1 + "vld1.16 {q0, q1}, [%[in]] \n" + + "vaddl.s16 q2, d0, d3 \n" // a0 = in[0] + in[12] + "vaddl.s16 q3, d1, d2 \n" // a1 = in[4] + in[8] + "vsubl.s16 q4, d1, d2 \n" // a2 = in[4] - in[8] + "vsubl.s16 q5, d0, d3 \n" // a3 = in[0] - in[12] + + "vadd.s32 q0, q2, q3 \n" // tmp[0] = a0 + a1 + "vsub.s32 q2, q2, q3 \n" // tmp[8] = a0 - a1 + "vadd.s32 q1, q5, q4 \n" // tmp[4] = a3 + a2 + "vsub.s32 q3, q5, q4 \n" // tmp[12] = a3 - a2 + + // Transpose + // q0 = tmp[0, 4, 8, 12], q1 = tmp[2, 6, 10, 14] + // q2 = tmp[1, 5, 9, 13], q3 = tmp[3, 7, 11, 15] + "vswp d1, d4 \n" // vtrn.64 q0, q2 + "vswp d3, d6 \n" // vtrn.64 q1, q3 + "vtrn.32 q0, q1 \n" + "vtrn.32 q2, q3 \n" + + "vmov.s32 q4, #3 \n" // dc = 3 + "vadd.s32 q0, q0, q4 \n" // dc = tmp[0] + 3 + "vadd.s32 q6, q0, q3 \n" // a0 = dc + tmp[3] + "vadd.s32 q7, q1, q2 \n" // a1 = tmp[1] + tmp[2] + "vsub.s32 q8, q1, q2 \n" // a2 = tmp[1] - tmp[2] + "vsub.s32 q9, q0, q3 \n" // a3 = dc - tmp[3] + + "vadd.s32 q0, q6, q7 \n" + "vshrn.s32 d0, q0, #3 \n" // (a0 + a1) >> 3 + "vadd.s32 q1, q9, q8 \n" + "vshrn.s32 d1, q1, #3 \n" // (a3 + a2) >> 3 + "vsub.s32 q2, q6, q7 \n" + "vshrn.s32 d2, q2, #3 \n" // (a0 - a1) >> 3 + "vsub.s32 q3, q9, q8 \n" + "vshrn.s32 d3, q3, #3 \n" // (a3 - a2) >> 3 + + // set the results to output + "vst1.16 d0[0], [%[out]], %[kStep] \n" + "vst1.16 d1[0], [%[out]], %[kStep] \n" + "vst1.16 d2[0], [%[out]], %[kStep] \n" + "vst1.16 d3[0], [%[out]], %[kStep] \n" + "vst1.16 d0[1], [%[out]], %[kStep] \n" + "vst1.16 d1[1], [%[out]], %[kStep] \n" + "vst1.16 d2[1], [%[out]], %[kStep] \n" + "vst1.16 d3[1], [%[out]], %[kStep] \n" + "vst1.16 d0[2], [%[out]], %[kStep] \n" + "vst1.16 d1[2], [%[out]], %[kStep] \n" + "vst1.16 d2[2], [%[out]], %[kStep] \n" + "vst1.16 d3[2], [%[out]], %[kStep] \n" + "vst1.16 d0[3], [%[out]], %[kStep] \n" + "vst1.16 d1[3], [%[out]], %[kStep] \n" + "vst1.16 d2[3], [%[out]], %[kStep] \n" + "vst1.16 d3[3], [%[out]], %[kStep] \n" + + : [out] "+r"(out) // modified registers + : [in] "r"(in), [kStep] "r"(kStep) // constants + : "memory", "q0", "q1", "q2", "q3", "q4", + "q5", "q6", "q7", "q8", "q9" // clobbered + ); +} + +// Forward transform. + +// adapted from vp8/encoder/arm/neon/shortfdct_neon.asm +static const int16_t kCoeff16[] = { + 5352, 5352, 5352, 5352, 2217, 2217, 2217, 2217 +}; +static const int32_t kCoeff32[] = { + 1812, 1812, 1812, 1812, + 937, 937, 937, 937, + 12000, 12000, 12000, 12000, + 51000, 51000, 51000, 51000 +}; + +static void FTransform(const uint8_t* src, const uint8_t* ref, + int16_t* out) { + const int kBPS = BPS; + const uint8_t* src_ptr = src; + const uint8_t* ref_ptr = ref; + const int16_t* coeff16 = kCoeff16; + const int32_t* coeff32 = kCoeff32; + + __asm__ volatile ( + // load src into q4, q5 in high half + "vld1.8 {d8}, [%[src_ptr]], %[kBPS] \n" + "vld1.8 {d10}, [%[src_ptr]], %[kBPS] \n" + "vld1.8 {d9}, [%[src_ptr]], %[kBPS] \n" + "vld1.8 {d11}, [%[src_ptr]] \n" + + // load ref into q6, q7 in high half + "vld1.8 {d12}, [%[ref_ptr]], %[kBPS] \n" + "vld1.8 {d14}, [%[ref_ptr]], %[kBPS] \n" + "vld1.8 {d13}, [%[ref_ptr]], %[kBPS] \n" + "vld1.8 {d15}, [%[ref_ptr]] \n" + + // Pack the high values in to q4 and q6 + "vtrn.32 q4, q5 \n" + "vtrn.32 q6, q7 \n" + + // d[0-3] = src - ref + "vsubl.u8 q0, d8, d12 \n" + "vsubl.u8 q1, d9, d13 \n" + + // load coeff16 into q8(d16=5352, d17=2217) + "vld1.16 {q8}, [%[coeff16]] \n" + + // load coeff32 high half into q9 = 1812, q10 = 937 + "vld1.32 {q9, q10}, [%[coeff32]]! \n" + + // load coeff32 low half into q11=12000, q12=51000 + "vld1.32 {q11,q12}, [%[coeff32]] \n" + + // part 1 + // Transpose. Register dN is the same as dN in C + "vtrn.32 d0, d2 \n" + "vtrn.32 d1, d3 \n" + "vtrn.16 d0, d1 \n" + "vtrn.16 d2, d3 \n" + + "vadd.s16 d4, d0, d3 \n" // a0 = d0 + d3 + "vadd.s16 d5, d1, d2 \n" // a1 = d1 + d2 + "vsub.s16 d6, d1, d2 \n" // a2 = d1 - d2 + "vsub.s16 d7, d0, d3 \n" // a3 = d0 - d3 + + "vadd.s16 d0, d4, d5 \n" // a0 + a1 + "vshl.s16 d0, d0, #3 \n" // temp[0+i*4] = (a0+a1) << 3 + "vsub.s16 d2, d4, d5 \n" // a0 - a1 + "vshl.s16 d2, d2, #3 \n" // (temp[2+i*4] = (a0-a1) << 3 + + "vmlal.s16 q9, d7, d16 \n" // a3*5352 + 1812 + "vmlal.s16 q10, d7, d17 \n" // a3*2217 + 937 + "vmlal.s16 q9, d6, d17 \n" // a2*2217 + a3*5352 + 1812 + "vmlsl.s16 q10, d6, d16 \n" // a3*2217 + 937 - a2*5352 + + // temp[1+i*4] = (d2*2217 + d3*5352 + 1812) >> 9 + // temp[3+i*4] = (d3*2217 + 937 - d2*5352) >> 9 + "vshrn.s32 d1, q9, #9 \n" + "vshrn.s32 d3, q10, #9 \n" + + // part 2 + // transpose d0=ip[0], d1=ip[4], d2=ip[8], d3=ip[12] + "vtrn.32 d0, d2 \n" + "vtrn.32 d1, d3 \n" + "vtrn.16 d0, d1 \n" + "vtrn.16 d2, d3 \n" + + "vmov.s16 d26, #7 \n" + + "vadd.s16 d4, d0, d3 \n" // a1 = ip[0] + ip[12] + "vadd.s16 d5, d1, d2 \n" // b1 = ip[4] + ip[8] + "vsub.s16 d6, d1, d2 \n" // c1 = ip[4] - ip[8] + "vadd.s16 d4, d4, d26 \n" // a1 + 7 + "vsub.s16 d7, d0, d3 \n" // d1 = ip[0] - ip[12] + + "vadd.s16 d0, d4, d5 \n" // op[0] = a1 + b1 + 7 + "vsub.s16 d2, d4, d5 \n" // op[8] = a1 - b1 + 7 + + "vmlal.s16 q11, d7, d16 \n" // d1*5352 + 12000 + "vmlal.s16 q12, d7, d17 \n" // d1*2217 + 51000 + + "vceq.s16 d4, d7, #0 \n" + + "vshr.s16 d0, d0, #4 \n" + "vshr.s16 d2, d2, #4 \n" + + "vmlal.s16 q11, d6, d17 \n" // c1*2217 + d1*5352 + 12000 + "vmlsl.s16 q12, d6, d16 \n" // d1*2217 - c1*5352 + 51000 + + "vmvn.s16 d4, d4 \n" + // op[4] = (c1*2217 + d1*5352 + 12000)>>16 + "vshrn.s32 d1, q11, #16 \n" + // op[4] += (d1!=0) + "vsub.s16 d1, d1, d4 \n" + // op[12]= (d1*2217 - c1*5352 + 51000)>>16 + "vshrn.s32 d3, q12, #16 \n" + + // set result to out array + "vst1.16 {q0, q1}, [%[out]] \n" + : [src_ptr] "+r"(src_ptr), [ref_ptr] "+r"(ref_ptr), + [coeff32] "+r"(coeff32) // modified registers + : [kBPS] "r"(kBPS), [coeff16] "r"(coeff16), + [out] "r"(out) // constants + : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", + "q10", "q11", "q12", "q13" // clobbered + ); +} + +static void FTransformWHT(const int16_t* in, int16_t* out) { + const int kStep = 32; + __asm__ volatile ( + // d0 = in[0 * 16] , d1 = in[1 * 16] + // d2 = in[2 * 16] , d3 = in[3 * 16] + "vld1.16 d0[0], [%[in]], %[kStep] \n" + "vld1.16 d1[0], [%[in]], %[kStep] \n" + "vld1.16 d2[0], [%[in]], %[kStep] \n" + "vld1.16 d3[0], [%[in]], %[kStep] \n" + "vld1.16 d0[1], [%[in]], %[kStep] \n" + "vld1.16 d1[1], [%[in]], %[kStep] \n" + "vld1.16 d2[1], [%[in]], %[kStep] \n" + "vld1.16 d3[1], [%[in]], %[kStep] \n" + "vld1.16 d0[2], [%[in]], %[kStep] \n" + "vld1.16 d1[2], [%[in]], %[kStep] \n" + "vld1.16 d2[2], [%[in]], %[kStep] \n" + "vld1.16 d3[2], [%[in]], %[kStep] \n" + "vld1.16 d0[3], [%[in]], %[kStep] \n" + "vld1.16 d1[3], [%[in]], %[kStep] \n" + "vld1.16 d2[3], [%[in]], %[kStep] \n" + "vld1.16 d3[3], [%[in]], %[kStep] \n" + + "vaddl.s16 q2, d0, d2 \n" + "vshl.s32 q2, q2, #2 \n" // a0=(in[0*16]+in[2*16])<<2 + "vaddl.s16 q3, d1, d3 \n" + "vshl.s32 q3, q3, #2 \n" // a1=(in[1*16]+in[3*16])<<2 + "vsubl.s16 q4, d1, d3 \n" + "vshl.s32 q4, q4, #2 \n" // a2=(in[1*16]-in[3*16])<<2 + "vsubl.s16 q5, d0, d2 \n" + "vshl.s32 q5, q5, #2 \n" // a3=(in[0*16]-in[2*16])<<2 + + "vceq.s32 q10, q2, #0 \n" + "vmvn.s32 q10, q10 \n" // (a0 != 0) + "vqadd.s32 q6, q2, q3 \n" // (a0 + a1) + "vqsub.s32 q6, q6, q10 \n" // (a0 + a1) + (a0 != 0) + "vqadd.s32 q7, q5, q4 \n" // a3 + a2 + "vqsub.s32 q8, q5, q4 \n" // a3 - a2 + "vqsub.s32 q9, q2, q3 \n" // a0 - a1 + + // Transpose + // q6 = tmp[0, 1, 2, 3] ; q7 = tmp[ 4, 5, 6, 7] + // q8 = tmp[8, 9, 10, 11] ; q9 = tmp[12, 13, 14, 15] + "vswp d13, d16 \n" // vtrn.64 q0, q2 + "vswp d15, d18 \n" // vtrn.64 q1, q3 + "vtrn.32 q6, q7 \n" + "vtrn.32 q8, q9 \n" + + "vqadd.s32 q0, q6, q8 \n" // a0 = tmp[0] + tmp[8] + "vqadd.s32 q1, q7, q9 \n" // a1 = tmp[4] + tmp[12] + "vqsub.s32 q2, q7, q9 \n" // a2 = tmp[4] - tmp[12] + "vqsub.s32 q3, q6, q8 \n" // a3 = tmp[0] - tmp[8] + + "vqadd.s32 q4, q0, q1 \n" // b0 = a0 + a1 + "vqadd.s32 q5, q3, q2 \n" // b1 = a3 + a2 + "vqsub.s32 q6, q3, q2 \n" // b2 = a3 - a2 + "vqsub.s32 q7, q0, q1 \n" // b3 = a0 - a1 + + "vmov.s32 q0, #3 \n" // q0 = 3 + + "vcgt.s32 q1, q4, #0 \n" // (b0>0) + "vqsub.s32 q2, q4, q1 \n" // (b0+(b0>0)) + "vqadd.s32 q3, q2, q0 \n" // (b0+(b0>0)+3) + "vshrn.s32 d18, q3, #3 \n" // (b0+(b0>0)+3) >> 3 + + "vcgt.s32 q1, q5, #0 \n" // (b1>0) + "vqsub.s32 q2, q5, q1 \n" // (b1+(b1>0)) + "vqadd.s32 q3, q2, q0 \n" // (b1+(b1>0)+3) + "vshrn.s32 d19, q3, #3 \n" // (b1+(b1>0)+3) >> 3 + + "vcgt.s32 q1, q6, #0 \n" // (b2>0) + "vqsub.s32 q2, q6, q1 \n" // (b2+(b2>0)) + "vqadd.s32 q3, q2, q0 \n" // (b2+(b2>0)+3) + "vshrn.s32 d20, q3, #3 \n" // (b2+(b2>0)+3) >> 3 + + "vcgt.s32 q1, q7, #0 \n" // (b3>0) + "vqsub.s32 q2, q7, q1 \n" // (b3+(b3>0)) + "vqadd.s32 q3, q2, q0 \n" // (b3+(b3>0)+3) + "vshrn.s32 d21, q3, #3 \n" // (b3+(b3>0)+3) >> 3 + + "vst1.16 {q9, q10}, [%[out]] \n" + + : [in] "+r"(in) + : [kStep] "r"(kStep), [out] "r"(out) + : "memory", "q0", "q1", "q2", "q3", "q4", "q5", + "q6", "q7", "q8", "q9", "q10" // clobbered + ) ; +} + +//------------------------------------------------------------------------------ +// Texture distortion +// +// We try to match the spectral content (weighted) between source and +// reconstructed samples. + +// Hadamard transform +// Returns the weighted sum of the absolute value of transformed coefficients. +// This uses a TTransform helper function in C +static int Disto4x4(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + const int kBPS = BPS; + const uint8_t* A = a; + const uint8_t* B = b; + const uint16_t* W = w; + int sum; + __asm__ volatile ( + "vld1.32 d0[0], [%[a]], %[kBPS] \n" + "vld1.32 d0[1], [%[a]], %[kBPS] \n" + "vld1.32 d2[0], [%[a]], %[kBPS] \n" + "vld1.32 d2[1], [%[a]] \n" + + "vld1.32 d1[0], [%[b]], %[kBPS] \n" + "vld1.32 d1[1], [%[b]], %[kBPS] \n" + "vld1.32 d3[0], [%[b]], %[kBPS] \n" + "vld1.32 d3[1], [%[b]] \n" + + // a d0/d2, b d1/d3 + // d0/d1: 01 01 01 01 + // d2/d3: 23 23 23 23 + // But: it goes 01 45 23 67 + // Notice the middle values are transposed + "vtrn.16 q0, q1 \n" + + // {a0, a1} = {in[0] + in[2], in[1] + in[3]} + "vaddl.u8 q2, d0, d2 \n" + "vaddl.u8 q10, d1, d3 \n" + // {a3, a2} = {in[0] - in[2], in[1] - in[3]} + "vsubl.u8 q3, d0, d2 \n" + "vsubl.u8 q11, d1, d3 \n" + + // tmp[0] = a0 + a1 + "vpaddl.s16 q0, q2 \n" + "vpaddl.s16 q8, q10 \n" + + // tmp[1] = a3 + a2 + "vpaddl.s16 q1, q3 \n" + "vpaddl.s16 q9, q11 \n" + + // No pair subtract + // q2 = {a0, a3} + // q3 = {a1, a2} + "vtrn.16 q2, q3 \n" + "vtrn.16 q10, q11 \n" + + // {tmp[3], tmp[2]} = {a0 - a1, a3 - a2} + "vsubl.s16 q12, d4, d6 \n" + "vsubl.s16 q13, d5, d7 \n" + "vsubl.s16 q14, d20, d22 \n" + "vsubl.s16 q15, d21, d23 \n" + + // separate tmp[3] and tmp[2] + // q12 = tmp[3] + // q13 = tmp[2] + "vtrn.32 q12, q13 \n" + "vtrn.32 q14, q15 \n" + + // Transpose tmp for a + "vswp d1, d26 \n" // vtrn.64 + "vswp d3, d24 \n" // vtrn.64 + "vtrn.32 q0, q1 \n" + "vtrn.32 q13, q12 \n" + + // Transpose tmp for b + "vswp d17, d30 \n" // vtrn.64 + "vswp d19, d28 \n" // vtrn.64 + "vtrn.32 q8, q9 \n" + "vtrn.32 q15, q14 \n" + + // The first Q register is a, the second b. + // q0/8 tmp[0-3] + // q13/15 tmp[4-7] + // q1/9 tmp[8-11] + // q12/14 tmp[12-15] + + // These are still in 01 45 23 67 order. We fix it easily in the addition + // case but the subtraction propegates them. + "vswp d3, d27 \n" + "vswp d19, d31 \n" + + // a0 = tmp[0] + tmp[8] + "vadd.s32 q2, q0, q1 \n" + "vadd.s32 q3, q8, q9 \n" + + // a1 = tmp[4] + tmp[12] + "vadd.s32 q10, q13, q12 \n" + "vadd.s32 q11, q15, q14 \n" + + // a2 = tmp[4] - tmp[12] + "vsub.s32 q13, q13, q12 \n" + "vsub.s32 q15, q15, q14 \n" + + // a3 = tmp[0] - tmp[8] + "vsub.s32 q0, q0, q1 \n" + "vsub.s32 q8, q8, q9 \n" + + // b0 = a0 + a1 + "vadd.s32 q1, q2, q10 \n" + "vadd.s32 q9, q3, q11 \n" + + // b1 = a3 + a2 + "vadd.s32 q12, q0, q13 \n" + "vadd.s32 q14, q8, q15 \n" + + // b2 = a3 - a2 + "vsub.s32 q0, q0, q13 \n" + "vsub.s32 q8, q8, q15 \n" + + // b3 = a0 - a1 + "vsub.s32 q2, q2, q10 \n" + "vsub.s32 q3, q3, q11 \n" + + "vld1.64 {q10, q11}, [%[w]] \n" + + // abs(b0) + "vabs.s32 q1, q1 \n" + "vabs.s32 q9, q9 \n" + // abs(b1) + "vabs.s32 q12, q12 \n" + "vabs.s32 q14, q14 \n" + // abs(b2) + "vabs.s32 q0, q0 \n" + "vabs.s32 q8, q8 \n" + // abs(b3) + "vabs.s32 q2, q2 \n" + "vabs.s32 q3, q3 \n" + + // expand w before using. + "vmovl.u16 q13, d20 \n" + "vmovl.u16 q15, d21 \n" + + // w[0] * abs(b0) + "vmul.u32 q1, q1, q13 \n" + "vmul.u32 q9, q9, q13 \n" + + // w[4] * abs(b1) + "vmla.u32 q1, q12, q15 \n" + "vmla.u32 q9, q14, q15 \n" + + // expand w before using. + "vmovl.u16 q13, d22 \n" + "vmovl.u16 q15, d23 \n" + + // w[8] * abs(b1) + "vmla.u32 q1, q0, q13 \n" + "vmla.u32 q9, q8, q13 \n" + + // w[12] * abs(b1) + "vmla.u32 q1, q2, q15 \n" + "vmla.u32 q9, q3, q15 \n" + + // Sum the arrays + "vpaddl.u32 q1, q1 \n" + "vpaddl.u32 q9, q9 \n" + "vadd.u64 d2, d3 \n" + "vadd.u64 d18, d19 \n" + + // Hadamard transform needs 4 bits of extra precision (2 bits in each + // direction) for dynamic raw. Weights w[] are 16bits at max, so the maximum + // precision for coeff is 8bit of input + 4bits of Hadamard transform + + // 16bits for w[] + 2 bits of abs() summation. + // + // This uses a maximum of 31 bits (signed). Discarding the top 32 bits is + // A-OK. + + // sum2 - sum1 + "vsub.u32 d0, d2, d18 \n" + // abs(sum2 - sum1) + "vabs.s32 d0, d0 \n" + // abs(sum2 - sum1) >> 5 + "vshr.u32 d0, #5 \n" + + // It would be better to move the value straight into r0 but I'm not + // entirely sure how this works with inline assembly. + "vmov.32 %[sum], d0[0] \n" + + : [sum] "=r"(sum), [a] "+r"(A), [b] "+r"(B), [w] "+r"(W) + : [kBPS] "r"(kBPS) + : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", + "q10", "q11", "q12", "q13", "q14", "q15" // clobbered + ) ; + + return sum; +} + +static int Disto16x16(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + int D = 0; + int x, y; + for (y = 0; y < 16 * BPS; y += 4 * BPS) { + for (x = 0; x < 16; x += 4) { + D += Disto4x4(a + x + y, b + x + y, w); + } + } + return D; +} + +#endif // WEBP_USE_NEON + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8EncDspInitNEON(void); + +void VP8EncDspInitNEON(void) { +#if defined(WEBP_USE_NEON) + VP8ITransform = ITransform; + VP8FTransform = FTransform; + + VP8ITransformWHT = ITransformWHT; + VP8FTransformWHT = FTransformWHT; + + VP8TDisto4x4 = Disto4x4; + VP8TDisto16x16 = Disto16x16; +#endif // WEBP_USE_NEON +} + +#if defined(__cplusplus) || defined(c_plusplus) +} // extern "C" +#endif diff --git a/third_party/libwebp/dsp/enc_sse2.c b/third_party/libwebp/dsp/enc_sse2.c index b046761..c4148b5 100644 --- a/third_party/libwebp/dsp/enc_sse2.c +++ b/third_party/libwebp/dsp/enc_sse2.c @@ -11,27 +11,58 @@ #include "./dsp.h" +#if defined(__cplusplus) || defined(c_plusplus) +extern "C" { +#endif + #if defined(WEBP_USE_SSE2) #include // for abs() #include #include "../enc/vp8enci.h" -#if defined(__cplusplus) || defined(c_plusplus) -extern "C" { +//------------------------------------------------------------------------------ +// Quite useful macro for debugging. Left here for convenience. + +#if 0 +#include +static void PrintReg(const __m128i r, const char* const name, int size) { + int n; + union { + __m128i r; + uint8_t i8[16]; + uint16_t i16[8]; + uint32_t i32[4]; + uint64_t i64[2]; + } tmp; + tmp.r = r; + printf("%s\t: ", name); + if (size == 8) { + for (n = 0; n < 16; ++n) printf("%.2x ", tmp.i8[n]); + } else if (size == 16) { + for (n = 0; n < 8; ++n) printf("%.4x ", tmp.i16[n]); + } else if (size == 32) { + for (n = 0; n < 4; ++n) printf("%.8x ", tmp.i32[n]); + } else { + for (n = 0; n < 2; ++n) printf("%.16lx ", tmp.i64[n]); + } + printf("\n"); +} #endif //------------------------------------------------------------------------------ // Compute susceptibility based on DCT-coeff histograms: // the higher, the "easier" the macroblock is to compress. -static int CollectHistogramSSE2(const uint8_t* ref, const uint8_t* pred, - int start_block, int end_block) { - int histo[MAX_COEFF_THRESH + 1] = { 0 }; - int16_t out[16]; - int j, k; +static void CollectHistogramSSE2(const uint8_t* ref, const uint8_t* pred, + int start_block, int end_block, + VP8Histogram* const histo) { const __m128i max_coeff_thresh = _mm_set1_epi16(MAX_COEFF_THRESH); + int j; for (j = start_block; j < end_block; ++j) { + int16_t out[16]; + int k; + VP8FTransform(ref + VP8DspScan[j], pred + VP8DspScan[j], out); // Convert coefficients to bin (within out[]). @@ -47,9 +78,9 @@ static int CollectHistogramSSE2(const uint8_t* ref, const uint8_t* pred, const __m128i xor1 = _mm_xor_si128(out1, sign1); const __m128i abs0 = _mm_sub_epi16(xor0, sign0); const __m128i abs1 = _mm_sub_epi16(xor1, sign1); - // v = abs(out) >> 2 - const __m128i v0 = _mm_srai_epi16(abs0, 2); - const __m128i v1 = _mm_srai_epi16(abs1, 2); + // v = abs(out) >> 3 + const __m128i v0 = _mm_srai_epi16(abs0, 3); + const __m128i v1 = _mm_srai_epi16(abs1, 3); // bin = min(v, MAX_COEFF_THRESH) const __m128i bin0 = _mm_min_epi16(v0, max_coeff_thresh); const __m128i bin1 = _mm_min_epi16(v1, max_coeff_thresh); @@ -58,13 +89,11 @@ static int CollectHistogramSSE2(const uint8_t* ref, const uint8_t* pred, _mm_storeu_si128((__m128i*)&out[8], bin1); } - // Use bin to update histogram. + // Convert coefficients to bin. for (k = 0; k < 16; ++k) { - histo[out[k]]++; + histo->distribution[out[k]]++; } } - - return VP8GetAlpha(histo); } //------------------------------------------------------------------------------ @@ -243,7 +272,7 @@ static void ITransformSSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst, // Add inverse transform to 'ref' and store. { - const __m128i zero = _mm_set1_epi16(0); + const __m128i zero = _mm_setzero_si128(); // Load the reference(s). __m128i ref0, ref1, ref2, ref3; if (do_two) { @@ -295,17 +324,23 @@ static void FTransformSSE2(const uint8_t* src, const uint8_t* ref, int16_t* out) { const __m128i zero = _mm_setzero_si128(); const __m128i seven = _mm_set1_epi16(7); - const __m128i k7500 = _mm_set1_epi32(7500); - const __m128i k14500 = _mm_set1_epi32(14500); + const __m128i k937 = _mm_set1_epi32(937); + const __m128i k1812 = _mm_set1_epi32(1812); const __m128i k51000 = _mm_set1_epi32(51000); const __m128i k12000_plus_one = _mm_set1_epi32(12000 + (1 << 16)); const __m128i k5352_2217 = _mm_set_epi16(5352, 2217, 5352, 2217, 5352, 2217, 5352, 2217); const __m128i k2217_5352 = _mm_set_epi16(2217, -5352, 2217, -5352, 2217, -5352, 2217, -5352); - + const __m128i k88p = _mm_set_epi16(8, 8, 8, 8, 8, 8, 8, 8); + const __m128i k88m = _mm_set_epi16(-8, 8, -8, 8, -8, 8, -8, 8); + const __m128i k5352_2217p = _mm_set_epi16(2217, 5352, 2217, 5352, + 2217, 5352, 2217, 5352); + const __m128i k5352_2217m = _mm_set_epi16(-5352, 2217, -5352, 2217, + -5352, 2217, -5352, 2217); __m128i v01, v32; + // Difference between src and ref and initial transpose. { // Load src and convert to 16b. @@ -326,73 +361,52 @@ static void FTransformSSE2(const uint8_t* src, const uint8_t* ref, const __m128i ref_1 = _mm_unpacklo_epi8(ref1, zero); const __m128i ref_2 = _mm_unpacklo_epi8(ref2, zero); const __m128i ref_3 = _mm_unpacklo_epi8(ref3, zero); - // Compute difference. + // Compute difference. -> 00 01 02 03 00 00 00 00 const __m128i diff0 = _mm_sub_epi16(src_0, ref_0); const __m128i diff1 = _mm_sub_epi16(src_1, ref_1); const __m128i diff2 = _mm_sub_epi16(src_2, ref_2); const __m128i diff3 = _mm_sub_epi16(src_3, ref_3); - // Transpose. + + // Unpack and shuffle // 00 01 02 03 0 0 0 0 // 10 11 12 13 0 0 0 0 // 20 21 22 23 0 0 0 0 // 30 31 32 33 0 0 0 0 - const __m128i transpose0_0 = _mm_unpacklo_epi16(diff0, diff1); - const __m128i transpose0_1 = _mm_unpacklo_epi16(diff2, diff3); - // 00 10 01 11 02 12 03 13 - // 20 30 21 31 22 32 23 33 - const __m128i v23 = _mm_unpackhi_epi32(transpose0_0, transpose0_1); - v01 = _mm_unpacklo_epi32(transpose0_0, transpose0_1); - v32 = _mm_shuffle_epi32(v23, _MM_SHUFFLE(1, 0, 3, 2)); - // a02 a12 a22 a32 a03 a13 a23 a33 - // a00 a10 a20 a30 a01 a11 a21 a31 - // a03 a13 a23 a33 a02 a12 a22 a32 - } - - // First pass and subsequent transpose. - { - // Same operations are done on the (0,3) and (1,2) pairs. - // b0 = (a0 + a3) << 3 - // b1 = (a1 + a2) << 3 - // b3 = (a0 - a3) << 3 - // b2 = (a1 - a2) << 3 - const __m128i a01 = _mm_add_epi16(v01, v32); - const __m128i a32 = _mm_sub_epi16(v01, v32); - const __m128i b01 = _mm_slli_epi16(a01, 3); - const __m128i b32 = _mm_slli_epi16(a32, 3); - const __m128i b11 = _mm_unpackhi_epi64(b01, b01); - const __m128i b22 = _mm_unpackhi_epi64(b32, b32); - - // e0 = b0 + b1 - // e2 = b0 - b1 - const __m128i e0 = _mm_add_epi16(b01, b11); - const __m128i e2 = _mm_sub_epi16(b01, b11); - const __m128i e02 = _mm_unpacklo_epi64(e0, e2); - - // e1 = (b3 * 5352 + b2 * 2217 + 14500) >> 12 - // e3 = (b3 * 2217 - b2 * 5352 + 7500) >> 12 - const __m128i b23 = _mm_unpacklo_epi16(b22, b32); - const __m128i c1 = _mm_madd_epi16(b23, k5352_2217); - const __m128i c3 = _mm_madd_epi16(b23, k2217_5352); - const __m128i d1 = _mm_add_epi32(c1, k14500); - const __m128i d3 = _mm_add_epi32(c3, k7500); - const __m128i e1 = _mm_srai_epi32(d1, 12); - const __m128i e3 = _mm_srai_epi32(d3, 12); - const __m128i e13 = _mm_packs_epi32(e1, e3); - - // Transpose. - // 00 01 02 03 20 21 22 23 - // 10 11 12 13 30 31 32 33 - const __m128i transpose0_0 = _mm_unpacklo_epi16(e02, e13); - const __m128i transpose0_1 = _mm_unpackhi_epi16(e02, e13); - // 00 10 01 11 02 12 03 13 - // 20 30 21 31 22 32 23 33 - const __m128i v23 = _mm_unpackhi_epi32(transpose0_0, transpose0_1); - v01 = _mm_unpacklo_epi32(transpose0_0, transpose0_1); - v32 = _mm_shuffle_epi32(v23, _MM_SHUFFLE(1, 0, 3, 2)); - // 02 12 22 32 03 13 23 33 - // 00 10 20 30 01 11 21 31 - // 03 13 23 33 02 12 22 32 + const __m128i shuf01 = _mm_unpacklo_epi32(diff0, diff1); + const __m128i shuf23 = _mm_unpacklo_epi32(diff2, diff3); + // 00 01 10 11 02 03 12 13 + // 20 21 30 31 22 23 32 33 + const __m128i shuf01_p = + _mm_shufflehi_epi16(shuf01, _MM_SHUFFLE(2, 3, 0, 1)); + const __m128i shuf23_p = + _mm_shufflehi_epi16(shuf23, _MM_SHUFFLE(2, 3, 0, 1)); + // 00 01 10 11 03 02 13 12 + // 20 21 30 31 23 22 33 32 + const __m128i s01 = _mm_unpacklo_epi64(shuf01_p, shuf23_p); + const __m128i s32 = _mm_unpackhi_epi64(shuf01_p, shuf23_p); + // 00 01 10 11 20 21 30 31 + // 03 02 13 12 23 22 33 32 + const __m128i a01 = _mm_add_epi16(s01, s32); + const __m128i a32 = _mm_sub_epi16(s01, s32); + // [d0 + d3 | d1 + d2 | ...] = [a0 a1 | a0' a1' | ... ] + // [d0 - d3 | d1 - d2 | ...] = [a3 a2 | a3' a2' | ... ] + + const __m128i tmp0 = _mm_madd_epi16(a01, k88p); // [ (a0 + a1) << 3, ... ] + const __m128i tmp2 = _mm_madd_epi16(a01, k88m); // [ (a0 - a1) << 3, ... ] + const __m128i tmp1_1 = _mm_madd_epi16(a32, k5352_2217p); + const __m128i tmp3_1 = _mm_madd_epi16(a32, k5352_2217m); + const __m128i tmp1_2 = _mm_add_epi32(tmp1_1, k1812); + const __m128i tmp3_2 = _mm_add_epi32(tmp3_1, k937); + const __m128i tmp1 = _mm_srai_epi32(tmp1_2, 9); + const __m128i tmp3 = _mm_srai_epi32(tmp3_2, 9); + const __m128i s03 = _mm_packs_epi32(tmp0, tmp2); + const __m128i s12 = _mm_packs_epi32(tmp1, tmp3); + const __m128i s_lo = _mm_unpacklo_epi16(s03, s12); // 0 1 0 1 0 1... + const __m128i s_hi = _mm_unpackhi_epi16(s03, s12); // 2 3 2 3 2 3 + const __m128i v23 = _mm_unpackhi_epi32(s_lo, s_hi); + v01 = _mm_unpacklo_epi32(s_lo, s_hi); + v32 = _mm_shuffle_epi32(v23, _MM_SHUFFLE(1, 0, 3, 2)); // 3 2 3 2 3 2.. } // Second pass @@ -406,13 +420,12 @@ static void FTransformSSE2(const uint8_t* src, const uint8_t* ref, const __m128i a32 = _mm_sub_epi16(v01, v32); const __m128i a11 = _mm_unpackhi_epi64(a01, a01); const __m128i a22 = _mm_unpackhi_epi64(a32, a32); + const __m128i a01_plus_7 = _mm_add_epi16(a01, seven); // d0 = (a0 + a1 + 7) >> 4; // d2 = (a0 - a1 + 7) >> 4; - const __m128i b0 = _mm_add_epi16(a01, a11); - const __m128i b2 = _mm_sub_epi16(a01, a11); - const __m128i c0 = _mm_add_epi16(b0, seven); - const __m128i c2 = _mm_add_epi16(b2, seven); + const __m128i c0 = _mm_add_epi16(a01_plus_7, a11); + const __m128i c2 = _mm_sub_epi16(a01_plus_7, a11); const __m128i d0 = _mm_srai_epi16(c0, 4); const __m128i d2 = _mm_srai_epi16(c2, 4); @@ -430,6 +443,7 @@ static void FTransformSSE2(const uint8_t* src, const uint8_t* ref, // f1 = f1 + (a3 != 0); // The compare will return (0xffff, 0) for (==0, !=0). To turn that into the // desired (0, 1), we add one earlier through k12000_plus_one. + // -> f1 = f1 + 1 - (a3 == 0) const __m128i g1 = _mm_add_epi16(f1, _mm_cmpeq_epi16(a32, zero)); _mm_storel_epi64((__m128i*)&out[ 0], d0); @@ -442,10 +456,101 @@ static void FTransformSSE2(const uint8_t* src, const uint8_t* ref, //------------------------------------------------------------------------------ // Metric +static int SSE_Nx4SSE2(const uint8_t* a, const uint8_t* b, + int num_quads, int do_16) { + const __m128i zero = _mm_setzero_si128(); + __m128i sum1 = zero; + __m128i sum2 = zero; + + while (num_quads-- > 0) { + // Note: for the !do_16 case, we read 16 pixels instead of 8 but that's ok, + // thanks to buffer over-allocation to that effect. + const __m128i a0 = _mm_loadu_si128((__m128i*)&a[BPS * 0]); + const __m128i a1 = _mm_loadu_si128((__m128i*)&a[BPS * 1]); + const __m128i a2 = _mm_loadu_si128((__m128i*)&a[BPS * 2]); + const __m128i a3 = _mm_loadu_si128((__m128i*)&a[BPS * 3]); + const __m128i b0 = _mm_loadu_si128((__m128i*)&b[BPS * 0]); + const __m128i b1 = _mm_loadu_si128((__m128i*)&b[BPS * 1]); + const __m128i b2 = _mm_loadu_si128((__m128i*)&b[BPS * 2]); + const __m128i b3 = _mm_loadu_si128((__m128i*)&b[BPS * 3]); + + // compute clip0(a-b) and clip0(b-a) + const __m128i a0p = _mm_subs_epu8(a0, b0); + const __m128i a0m = _mm_subs_epu8(b0, a0); + const __m128i a1p = _mm_subs_epu8(a1, b1); + const __m128i a1m = _mm_subs_epu8(b1, a1); + const __m128i a2p = _mm_subs_epu8(a2, b2); + const __m128i a2m = _mm_subs_epu8(b2, a2); + const __m128i a3p = _mm_subs_epu8(a3, b3); + const __m128i a3m = _mm_subs_epu8(b3, a3); + + // compute |a-b| with 8b arithmetic as clip0(a-b) | clip0(b-a) + const __m128i diff0 = _mm_or_si128(a0p, a0m); + const __m128i diff1 = _mm_or_si128(a1p, a1m); + const __m128i diff2 = _mm_or_si128(a2p, a2m); + const __m128i diff3 = _mm_or_si128(a3p, a3m); + + // unpack (only four operations, instead of eight) + const __m128i low0 = _mm_unpacklo_epi8(diff0, zero); + const __m128i low1 = _mm_unpacklo_epi8(diff1, zero); + const __m128i low2 = _mm_unpacklo_epi8(diff2, zero); + const __m128i low3 = _mm_unpacklo_epi8(diff3, zero); + + // multiply with self + const __m128i low_madd0 = _mm_madd_epi16(low0, low0); + const __m128i low_madd1 = _mm_madd_epi16(low1, low1); + const __m128i low_madd2 = _mm_madd_epi16(low2, low2); + const __m128i low_madd3 = _mm_madd_epi16(low3, low3); + + // collect in a cascading way + const __m128i low_sum0 = _mm_add_epi32(low_madd0, low_madd1); + const __m128i low_sum1 = _mm_add_epi32(low_madd2, low_madd3); + sum1 = _mm_add_epi32(sum1, low_sum0); + sum2 = _mm_add_epi32(sum2, low_sum1); + + if (do_16) { // if necessary, process the higher 8 bytes similarly + const __m128i hi0 = _mm_unpackhi_epi8(diff0, zero); + const __m128i hi1 = _mm_unpackhi_epi8(diff1, zero); + const __m128i hi2 = _mm_unpackhi_epi8(diff2, zero); + const __m128i hi3 = _mm_unpackhi_epi8(diff3, zero); + + const __m128i hi_madd0 = _mm_madd_epi16(hi0, hi0); + const __m128i hi_madd1 = _mm_madd_epi16(hi1, hi1); + const __m128i hi_madd2 = _mm_madd_epi16(hi2, hi2); + const __m128i hi_madd3 = _mm_madd_epi16(hi3, hi3); + const __m128i hi_sum0 = _mm_add_epi32(hi_madd0, hi_madd1); + const __m128i hi_sum1 = _mm_add_epi32(hi_madd2, hi_madd3); + sum1 = _mm_add_epi32(sum1, hi_sum0); + sum2 = _mm_add_epi32(sum2, hi_sum1); + } + a += 4 * BPS; + b += 4 * BPS; + } + { + int32_t tmp[4]; + const __m128i sum = _mm_add_epi32(sum1, sum2); + _mm_storeu_si128((__m128i*)tmp, sum); + return (tmp[3] + tmp[2] + tmp[1] + tmp[0]); + } +} + +static int SSE16x16SSE2(const uint8_t* a, const uint8_t* b) { + return SSE_Nx4SSE2(a, b, 4, 1); +} + +static int SSE16x8SSE2(const uint8_t* a, const uint8_t* b) { + return SSE_Nx4SSE2(a, b, 2, 1); +} + +static int SSE8x8SSE2(const uint8_t* a, const uint8_t* b) { + return SSE_Nx4SSE2(a, b, 2, 0); +} + static int SSE4x4SSE2(const uint8_t* a, const uint8_t* b) { - const __m128i zero = _mm_set1_epi16(0); + const __m128i zero = _mm_setzero_si128(); - // Load values. + // Load values. Note that we read 8 pixels instead of 4, + // but the a/b buffers are over-allocated to that effect. const __m128i a0 = _mm_loadl_epi64((__m128i*)&a[BPS * 0]); const __m128i a1 = _mm_loadl_epi64((__m128i*)&a[BPS * 1]); const __m128i a2 = _mm_loadl_epi64((__m128i*)&a[BPS * 2]); @@ -483,6 +588,7 @@ static int SSE4x4SSE2(const uint8_t* a, const uint8_t* b) { const __m128i sum0 = _mm_add_epi32(madd0, madd1); const __m128i sum1 = _mm_add_epi32(madd2, madd3); const __m128i sum2 = _mm_add_epi32(sum0, sum1); + int32_t tmp[4]; _mm_storeu_si128((__m128i*)tmp, sum2); return (tmp[3] + tmp[2] + tmp[1] + tmp[0]); @@ -502,8 +608,6 @@ static int TTransformSSE2(const uint8_t* inA, const uint8_t* inB, int32_t sum[4]; __m128i tmp_0, tmp_1, tmp_2, tmp_3; const __m128i zero = _mm_setzero_si128(); - const __m128i one = _mm_set1_epi16(1); - const __m128i three = _mm_set1_epi16(3); // Load, combine and tranpose inputs. { @@ -550,17 +654,14 @@ static int TTransformSSE2(const uint8_t* inA, const uint8_t* inB, // Horizontal pass and subsequent transpose. { // Calculate a and b (two 4x4 at once). - const __m128i a0 = _mm_slli_epi16(_mm_add_epi16(tmp_0, tmp_2), 2); - const __m128i a1 = _mm_slli_epi16(_mm_add_epi16(tmp_1, tmp_3), 2); - const __m128i a2 = _mm_slli_epi16(_mm_sub_epi16(tmp_1, tmp_3), 2); - const __m128i a3 = _mm_slli_epi16(_mm_sub_epi16(tmp_0, tmp_2), 2); - // b0_extra = (a0 != 0); - const __m128i b0_extra = _mm_andnot_si128(_mm_cmpeq_epi16 (a0, zero), one); - const __m128i b0_base = _mm_add_epi16(a0, a1); + const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2); + const __m128i a1 = _mm_add_epi16(tmp_1, tmp_3); + const __m128i a2 = _mm_sub_epi16(tmp_1, tmp_3); + const __m128i a3 = _mm_sub_epi16(tmp_0, tmp_2); + const __m128i b0 = _mm_add_epi16(a0, a1); const __m128i b1 = _mm_add_epi16(a3, a2); const __m128i b2 = _mm_sub_epi16(a3, a2); const __m128i b3 = _mm_sub_epi16(a0, a1); - const __m128i b0 = _mm_add_epi16(b0_base, b0_extra); // a00 a01 a02 a03 b00 b01 b02 b03 // a10 a11 a12 a13 b10 b11 b12 b13 // a20 a21 a22 a23 b20 b21 b22 b23 @@ -635,19 +736,6 @@ static int TTransformSSE2(const uint8_t* inA, const uint8_t* inB, B_b2 = _mm_sub_epi16(B_b2, sign_B_b2); } - // b = abs(b) + 3 - A_b0 = _mm_add_epi16(A_b0, three); - A_b2 = _mm_add_epi16(A_b2, three); - B_b0 = _mm_add_epi16(B_b0, three); - B_b2 = _mm_add_epi16(B_b2, three); - - // abs((b + (b<0) + 3) >> 3) = (abs(b) + 3) >> 3 - // b = (abs(b) + 3) >> 3 - A_b0 = _mm_srai_epi16(A_b0, 3); - A_b2 = _mm_srai_epi16(A_b2, 3); - B_b0 = _mm_srai_epi16(B_b0, 3); - B_b2 = _mm_srai_epi16(B_b2, 3); - // weighted sums A_b0 = _mm_madd_epi16(A_b0, w_0); A_b2 = _mm_madd_epi16(A_b2, w_8); @@ -666,7 +754,7 @@ static int TTransformSSE2(const uint8_t* inA, const uint8_t* inB, static int Disto4x4SSE2(const uint8_t* const a, const uint8_t* const b, const uint16_t* const w) { const int diff_sum = TTransformSSE2(a, b, w); - return (abs(diff_sum) + 8) >> 4; + return abs(diff_sum) >> 5; } static int Disto16x16SSE2(const uint8_t* const a, const uint8_t* const b, @@ -681,7 +769,6 @@ static int Disto16x16SSE2(const uint8_t* const a, const uint8_t* const b, return D; } - //------------------------------------------------------------------------------ // Quantization // @@ -690,8 +777,7 @@ static int Disto16x16SSE2(const uint8_t* const a, const uint8_t* const b, static int QuantizeBlockSSE2(int16_t in[16], int16_t out[16], int n, const VP8Matrix* const mtx) { const __m128i max_coeff_2047 = _mm_set1_epi16(2047); - const __m128i zero = _mm_set1_epi16(0); - __m128i sign0, sign8; + const __m128i zero = _mm_setzero_si128(); __m128i coeff0, coeff8; __m128i out0, out8; __m128i packed_out; @@ -713,8 +799,8 @@ static int QuantizeBlockSSE2(int16_t in[16], int16_t out[16], const __m128i zthresh8 = _mm_loadu_si128((__m128i*)&mtx->zthresh_[8]); // sign(in) = in >> 15 (0x0000 if positive, 0xffff if negative) - sign0 = _mm_srai_epi16(in0, 15); - sign8 = _mm_srai_epi16(in8, 15); + const __m128i sign0 = _mm_srai_epi16(in0, 15); + const __m128i sign8 = _mm_srai_epi16(in8, 15); // coeff = abs(in) = (in ^ sign) - sign coeff0 = _mm_xor_si128(in0, sign0); @@ -819,19 +905,28 @@ static int QuantizeBlockSSE2(int16_t in[16], int16_t out[16], } } +#endif // WEBP_USE_SSE2 + +//------------------------------------------------------------------------------ +// Entry point + extern void VP8EncDspInitSSE2(void); + void VP8EncDspInitSSE2(void) { +#if defined(WEBP_USE_SSE2) VP8CollectHistogram = CollectHistogramSSE2; VP8EncQuantizeBlock = QuantizeBlockSSE2; VP8ITransform = ITransformSSE2; VP8FTransform = FTransformSSE2; + VP8SSE16x16 = SSE16x16SSE2; + VP8SSE16x8 = SSE16x8SSE2; + VP8SSE8x8 = SSE8x8SSE2; VP8SSE4x4 = SSE4x4SSE2; VP8TDisto4x4 = Disto4x4SSE2; VP8TDisto16x16 = Disto16x16SSE2; +#endif // WEBP_USE_SSE2 } #if defined(__cplusplus) || defined(c_plusplus) } // extern "C" #endif - -#endif // WEBP_USE_SSE2 diff --git a/third_party/libwebp/dsp/lossless.c b/third_party/libwebp/dsp/lossless.c index 472e641..080b3e6 100644 --- a/third_party/libwebp/dsp/lossless.c +++ b/third_party/libwebp/dsp/lossless.c @@ -11,25 +11,31 @@ // Jyrki Alakuijala (jyrki@google.com) // Urvang Joshi (urvang@google.com) +#include "./dsp.h" + +// Define the following if target arch is sure to have SSE2 +// #define WEBP_TARGET_HAS_SSE2 + #if defined(__cplusplus) || defined(c_plusplus) extern "C" { #endif +#if defined(WEBP_TARGET_HAS_SSE2) +#include +#endif + #include #include #include "./lossless.h" #include "../dec/vp8li.h" -#include "../dsp/yuv.h" -#include "../dsp/dsp.h" -#include "../enc/histogram.h" +#include "./yuv.h" #define MAX_DIFF_COST (1e30f) // lookup table for small values of log2(int) #define APPROX_LOG_MAX 4096 #define LOG_2_RECIPROCAL 1.44269504088896338700465094007086 -#define LOG_LOOKUP_IDX_MAX 256 -static const float kLog2Table[LOG_LOOKUP_IDX_MAX] = { +const float kLog2Table[LOG_LOOKUP_IDX_MAX] = { 0.0000000000000000f, 0.0000000000000000f, 1.0000000000000000f, 1.5849625007211560f, 2.0000000000000000f, 2.3219280948873621f, @@ -160,16 +166,97 @@ static const float kLog2Table[LOG_LOOKUP_IDX_MAX] = { 7.9886846867721654f, 7.9943534368588577f }; -float VP8LFastLog2(int v) { - if (v < LOG_LOOKUP_IDX_MAX) { - return kLog2Table[v]; - } else if (v < APPROX_LOG_MAX) { +const float kSLog2Table[LOG_LOOKUP_IDX_MAX] = { + 0.00000000f, 0.00000000f, 2.00000000f, 4.75488750f, + 8.00000000f, 11.60964047f, 15.50977500f, 19.65148445f, + 24.00000000f, 28.52932501f, 33.21928095f, 38.05374781f, + 43.01955001f, 48.10571634f, 53.30296891f, 58.60335893f, + 64.00000000f, 69.48686830f, 75.05865003f, 80.71062276f, + 86.43856190f, 92.23866588f, 98.10749561f, 104.04192499f, + 110.03910002f, 116.09640474f, 122.21143267f, 128.38196256f, + 134.60593782f, 140.88144886f, 147.20671787f, 153.58008562f, + 160.00000000f, 166.46500594f, 172.97373660f, 179.52490559f, + 186.11730005f, 192.74977453f, 199.42124551f, 206.13068654f, + 212.87712380f, 219.65963219f, 226.47733176f, 233.32938445f, + 240.21499122f, 247.13338933f, 254.08384998f, 261.06567603f, + 268.07820003f, 275.12078236f, 282.19280949f, 289.29369244f, + 296.42286534f, 303.57978409f, 310.76392512f, 317.97478424f, + 325.21187564f, 332.47473081f, 339.76289772f, 347.07593991f, + 354.41343574f, 361.77497759f, 369.16017124f, 376.56863518f, + 384.00000000f, 391.45390785f, 398.93001188f, 406.42797576f, + 413.94747321f, 421.48818752f, 429.04981119f, 436.63204548f, + 444.23460010f, 451.85719280f, 459.49954906f, 467.16140179f, + 474.84249102f, 482.54256363f, 490.26137307f, 497.99867911f, + 505.75424759f, 513.52785023f, 521.31926438f, 529.12827280f, + 536.95466351f, 544.79822957f, 552.65876890f, 560.53608414f, + 568.42998244f, 576.34027536f, 584.26677867f, 592.20931226f, + 600.16769996f, 608.14176943f, 616.13135206f, 624.13628279f, + 632.15640007f, 640.19154569f, 648.24156472f, 656.30630539f, + 664.38561898f, 672.47935976f, 680.58738488f, 688.70955430f, + 696.84573069f, 704.99577935f, 713.15956818f, 721.33696754f, + 729.52785023f, 737.73209140f, 745.94956849f, 754.18016116f, + 762.42375127f, 770.68022275f, 778.94946161f, 787.23135586f, + 795.52579543f, 803.83267219f, 812.15187982f, 820.48331383f, + 828.82687147f, 837.18245171f, 845.54995518f, 853.92928416f, + 862.32034249f, 870.72303558f, 879.13727036f, 887.56295522f, + 896.00000000f, 904.44831595f, 912.90781569f, 921.37841320f, + 929.86002376f, 938.35256392f, 946.85595152f, 955.37010560f, + 963.89494641f, 972.43039537f, 980.97637504f, 989.53280911f, + 998.09962237f, 1006.67674069f, 1015.26409097f, 1023.86160116f, + 1032.46920021f, 1041.08681805f, 1049.71438560f, 1058.35183469f, + 1066.99909811f, 1075.65610955f, 1084.32280357f, 1092.99911564f, + 1101.68498204f, 1110.38033993f, 1119.08512727f, 1127.79928282f, + 1136.52274614f, 1145.25545758f, 1153.99735821f, 1162.74838989f, + 1171.50849518f, 1180.27761738f, 1189.05570047f, 1197.84268914f, + 1206.63852876f, 1215.44316535f, 1224.25654560f, 1233.07861684f, + 1241.90932703f, 1250.74862473f, 1259.59645914f, 1268.45278005f, + 1277.31753781f, 1286.19068338f, 1295.07216828f, 1303.96194457f, + 1312.85996488f, 1321.76618236f, 1330.68055071f, 1339.60302413f, + 1348.53355734f, 1357.47210556f, 1366.41862452f, 1375.37307041f, + 1384.33539991f, 1393.30557020f, 1402.28353887f, 1411.26926400f, + 1420.26270412f, 1429.26381818f, 1438.27256558f, 1447.28890615f, + 1456.31280014f, 1465.34420819f, 1474.38309138f, 1483.42941118f, + 1492.48312945f, 1501.54420843f, 1510.61261078f, 1519.68829949f, + 1528.77123795f, 1537.86138993f, 1546.95871952f, 1556.06319119f, + 1565.17476976f, 1574.29342040f, 1583.41910860f, 1592.55180020f, + 1601.69146137f, 1610.83805860f, 1619.99155871f, 1629.15192882f, + 1638.31913637f, 1647.49314911f, 1656.67393509f, 1665.86146266f, + 1675.05570047f, 1684.25661744f, 1693.46418280f, 1702.67836605f, + 1711.89913698f, 1721.12646563f, 1730.36032233f, 1739.60067768f, + 1748.84750254f, 1758.10076802f, 1767.36044551f, 1776.62650662f, + 1785.89892323f, 1795.17766747f, 1804.46271172f, 1813.75402857f, + 1823.05159087f, 1832.35537170f, 1841.66534438f, 1850.98148244f, + 1860.30375965f, 1869.63214999f, 1878.96662767f, 1888.30716711f, + 1897.65374295f, 1907.00633003f, 1916.36490342f, 1925.72943838f, + 1935.09991037f, 1944.47629506f, 1953.85856831f, 1963.24670620f, + 1972.64068498f, 1982.04048108f, 1991.44607117f, 2000.85743204f, + 2010.27454072f, 2019.69737440f, 2029.12591044f, 2038.56012640f +}; + +float VP8LFastSLog2Slow(int v) { + assert(v >= LOG_LOOKUP_IDX_MAX); + if (v < APPROX_LOG_MAX) { int log_cnt = 0; + const float v_f = (float)v; while (v >= LOG_LOOKUP_IDX_MAX) { ++log_cnt; v = v >> 1; } - return kLog2Table[v] + (float)log_cnt; + return v_f * (kLog2Table[v] + log_cnt); + } else { + return (float)(LOG_2_RECIPROCAL * v * log((double)v)); + } +} + +float VP8LFastLog2Slow(int v) { + assert(v >= LOG_LOOKUP_IDX_MAX); + if (v < APPROX_LOG_MAX) { + int log_cnt = 0; + while (v >= LOG_LOOKUP_IDX_MAX) { + ++log_cnt; + v = v >> 1; + } + return kLog2Table[v] + log_cnt; } else { return (float)(LOG_2_RECIPROCAL * log((double)v)); } @@ -198,6 +285,61 @@ static WEBP_INLINE uint32_t Average4(uint32_t a0, uint32_t a1, return Average2(Average2(a0, a1), Average2(a2, a3)); } +#if defined(WEBP_TARGET_HAS_SSE2) +static WEBP_INLINE uint32_t ClampedAddSubtractFull(uint32_t c0, uint32_t c1, + uint32_t c2) { + const __m128i zero = _mm_setzero_si128(); + const __m128i C0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c0), zero); + const __m128i C1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c1), zero); + const __m128i C2 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c2), zero); + const __m128i V1 = _mm_add_epi16(C0, C1); + const __m128i V2 = _mm_sub_epi16(V1, C2); + const __m128i b = _mm_packus_epi16(V2, V2); + const uint32_t output = _mm_cvtsi128_si32(b); + return output; +} + +static WEBP_INLINE uint32_t ClampedAddSubtractHalf(uint32_t c0, uint32_t c1, + uint32_t c2) { + const uint32_t ave = Average2(c0, c1); + const __m128i zero = _mm_setzero_si128(); + const __m128i A0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(ave), zero); + const __m128i B0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c2), zero); + const __m128i A1 = _mm_sub_epi16(A0, B0); + const __m128i BgtA = _mm_cmpgt_epi16(B0, A0); + const __m128i A2 = _mm_sub_epi16(A1, BgtA); + const __m128i A3 = _mm_srai_epi16(A2, 1); + const __m128i A4 = _mm_add_epi16(A0, A3); + const __m128i A5 = _mm_packus_epi16(A4, A4); + const uint32_t output = _mm_cvtsi128_si32(A5); + return output; +} + +static WEBP_INLINE uint32_t Select(uint32_t a, uint32_t b, uint32_t c) { + int pa_minus_pb; + const __m128i zero = _mm_setzero_si128(); + const __m128i A0 = _mm_cvtsi32_si128(a); + const __m128i B0 = _mm_cvtsi32_si128(b); + const __m128i C0 = _mm_cvtsi32_si128(c); + const __m128i AC0 = _mm_subs_epu8(A0, C0); + const __m128i CA0 = _mm_subs_epu8(C0, A0); + const __m128i BC0 = _mm_subs_epu8(B0, C0); + const __m128i CB0 = _mm_subs_epu8(C0, B0); + const __m128i AC = _mm_or_si128(AC0, CA0); + const __m128i BC = _mm_or_si128(BC0, CB0); + const __m128i pa = _mm_unpacklo_epi8(AC, zero); // |a - c| + const __m128i pb = _mm_unpacklo_epi8(BC, zero); // |b - c| + const __m128i diff = _mm_sub_epi16(pb, pa); + { + int16_t out[8]; + _mm_storeu_si128((__m128i*)out, diff); + pa_minus_pb = out[0] + out[1] + out[2] + out[3]; + } + return (pa_minus_pb <= 0) ? a : b; +} + +#else + static WEBP_INLINE uint32_t Clip255(uint32_t a) { if (a < 256) { return a; @@ -239,9 +381,9 @@ static WEBP_INLINE uint32_t ClampedAddSubtractHalf(uint32_t c0, uint32_t c1, } static WEBP_INLINE int Sub3(int a, int b, int c) { - const int pa = b - c; - const int pb = a - c; - return abs(pa) - abs(pb); + const int pb = b - c; + const int pa = a - c; + return abs(pb) - abs(pa); } static WEBP_INLINE uint32_t Select(uint32_t a, uint32_t b, uint32_t c) { @@ -250,9 +392,9 @@ static WEBP_INLINE uint32_t Select(uint32_t a, uint32_t b, uint32_t c) { Sub3((a >> 16) & 0xff, (b >> 16) & 0xff, (c >> 16) & 0xff) + Sub3((a >> 8) & 0xff, (b >> 8) & 0xff, (c >> 8) & 0xff) + Sub3((a ) & 0xff, (b ) & 0xff, (c ) & 0xff); - return (pa_minus_pb <= 0) ? a : b; } +#endif //------------------------------------------------------------------------------ // Predictors @@ -340,35 +482,36 @@ static float PredictionCostSpatial(const int* counts, return (float)(-0.1 * bits); } -// Compute the Shanon's entropy: Sum(p*log2(p)) -static float ShannonEntropy(const int* const array, int n) { +// Compute the combined Shanon's entropy for distribution {X} and {X+Y} +static float CombinedShannonEntropy(const int* const X, + const int* const Y, int n) { int i; - float retval = 0.f; - int sum = 0; + double retval = 0.; + int sumX = 0, sumXY = 0; for (i = 0; i < n; ++i) { - if (array[i] != 0) { - sum += array[i]; - retval -= VP8LFastSLog2(array[i]); + const int x = X[i]; + const int xy = X[i] + Y[i]; + if (x != 0) { + sumX += x; + retval -= VP8LFastSLog2(x); + } + if (xy != 0) { + sumXY += xy; + retval -= VP8LFastSLog2(xy); } } - retval += VP8LFastSLog2(sum); - return retval; + retval += VP8LFastSLog2(sumX) + VP8LFastSLog2(sumXY); + return (float)retval; } static float PredictionCostSpatialHistogram(int accumulated[4][256], int tile[4][256]) { int i; - int k; - int combo[256]; double retval = 0; for (i = 0; i < 4; ++i) { - const double exp_val = 0.94; - retval += PredictionCostSpatial(&tile[i][0], 1, exp_val); - retval += ShannonEntropy(&tile[i][0], 256); - for (k = 0; k < 256; ++k) { - combo[k] = accumulated[i][k] + tile[i][k]; - } - retval += ShannonEntropy(&combo[0], 256); + const double kExpValue = 0.94; + retval += PredictionCostSpatial(tile[i], 1, kExpValue); + retval += CombinedShannonEntropy(tile[i], accumulated[i], 256); } return (float)retval; } @@ -572,8 +715,21 @@ static void PredictorInverseTransform(const VP8LTransform* const transform, } void VP8LSubtractGreenFromBlueAndRed(uint32_t* argb_data, int num_pixs) { - int i; - for (i = 0; i < num_pixs; ++i) { + int i = 0; +#if defined(WEBP_TARGET_HAS_SSE2) + const __m128i mask = _mm_set1_epi32(0x0000ff00); + for (; i + 4 < num_pixs; i += 4) { + const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); + const __m128i in_00g0 = _mm_and_si128(in, mask); // 00g0|00g0|... + const __m128i in_0g00 = _mm_slli_epi32(in_00g0, 8); // 0g00|0g00|... + const __m128i in_000g = _mm_srli_epi32(in_00g0, 8); // 000g|000g|... + const __m128i in_0g0g = _mm_or_si128(in_0g00, in_000g); + const __m128i out = _mm_sub_epi8(in, in_0g0g); + _mm_storeu_si128((__m128i*)&argb_data[i], out); + } + // fallthrough and finish off with plain-C +#endif + for (; i < num_pixs; ++i) { const uint32_t argb = argb_data[i]; const uint32_t green = (argb >> 8) & 0xff; const uint32_t new_r = (((argb >> 16) & 0xff) - green) & 0xff; @@ -588,9 +744,21 @@ static void AddGreenToBlueAndRed(const VP8LTransform* const transform, int y_start, int y_end, uint32_t* data) { const int width = transform->xsize_; const uint32_t* const data_end = data + (y_end - y_start) * width; +#if defined(WEBP_TARGET_HAS_SSE2) + const __m128i mask = _mm_set1_epi32(0x0000ff00); + for (; data + 4 < data_end; data += 4) { + const __m128i in = _mm_loadu_si128((__m128i*)data); + const __m128i in_00g0 = _mm_and_si128(in, mask); // 00g0|00g0|... + const __m128i in_0g00 = _mm_slli_epi32(in_00g0, 8); // 0g00|0g00|... + const __m128i in_000g = _mm_srli_epi32(in_00g0, 8); // 000g|000g|... + const __m128i in_0g0g = _mm_or_si128(in_0g00, in_000g); + const __m128i out = _mm_add_epi8(in, in_0g0g); + _mm_storeu_si128((__m128i*)data, out); + } + // fallthrough and finish off with plain-C +#endif while (data < data_end) { const uint32_t argb = *data; - // "* 0001001u" is equivalent to "(green << 16) + green)" const uint32_t green = ((argb >> 8) & 0xff); uint32_t red_blue = (argb & 0x00ff00ffu); red_blue += (green << 16) | green; @@ -655,6 +823,25 @@ static WEBP_INLINE uint32_t TransformColor(const Multipliers* const m, return (argb & 0xff00ff00u) | (new_red << 16) | (new_blue); } +static WEBP_INLINE uint8_t TransformColorRed(uint8_t green_to_red, + uint32_t argb) { + const uint32_t green = argb >> 8; + uint32_t new_red = argb >> 16; + new_red -= ColorTransformDelta(green_to_red, green); + return (new_red & 0xff); +} + +static WEBP_INLINE uint8_t TransformColorBlue(uint8_t green_to_blue, + uint8_t red_to_blue, + uint32_t argb) { + const uint32_t green = argb >> 8; + const uint32_t red = argb >> 16; + uint8_t new_blue = argb; + new_blue -= ColorTransformDelta(green_to_blue, green); + new_blue -= ColorTransformDelta(red_to_blue, red); + return (new_blue & 0xff); +} + static WEBP_INLINE int SkipRepeatedPixels(const uint32_t* const argb, int ix, int xsize) { const uint32_t v = argb[ix]; @@ -675,14 +862,10 @@ static WEBP_INLINE int SkipRepeatedPixels(const uint32_t* const argb, static float PredictionCostCrossColor(const int accumulated[256], const int counts[256]) { // Favor low entropy, locally and globally. - int i; - int combo[256]; - for (i = 0; i < 256; ++i) { - combo[i] = accumulated[i] + counts[i]; - } - return ShannonEntropy(combo, 256) + - ShannonEntropy(counts, 256) + - PredictionCostSpatial(counts, 3, 2.4); // Favor small absolute values. + // Favor small absolute values for PredictionCostSpatial + static const double kExpValue = 2.4; + return CombinedShannonEntropy(counts, accumulated, 256) + + PredictionCostSpatial(counts, 3, kExpValue); } static Multipliers GetBestColorTransformForTile( @@ -712,85 +895,75 @@ static Multipliers GetBestColorTransformForTile( if (all_y_max > ysize) { all_y_max = ysize; } + for (green_to_red = -64; green_to_red <= 64; green_to_red += halfstep) { int histo[256] = { 0 }; int all_y; - Multipliers tx; - MultipliersClear(&tx); - tx.green_to_red_ = green_to_red & 0xff; for (all_y = tile_y_offset; all_y < all_y_max; ++all_y) { - uint32_t predict; int ix = all_y * xsize + tile_x_offset; int all_x; for (all_x = tile_x_offset; all_x < all_x_max; ++all_x, ++ix) { if (SkipRepeatedPixels(argb, ix, xsize)) { continue; } - predict = TransformColor(&tx, argb[ix], 0); - ++histo[(predict >> 16) & 0xff]; // red. + ++histo[TransformColorRed(green_to_red, argb[ix])]; // red. } } cur_diff = PredictionCostCrossColor(&accumulated_red_histo[0], &histo[0]); - if (tx.green_to_red_ == prevX.green_to_red_) { + if ((uint8_t)green_to_red == prevX.green_to_red_) { cur_diff -= 3; // favor keeping the areas locally similar } - if (tx.green_to_red_ == prevY.green_to_red_) { + if ((uint8_t)green_to_red == prevY.green_to_red_) { cur_diff -= 3; // favor keeping the areas locally similar } - if (tx.green_to_red_ == 0) { + if (green_to_red == 0) { cur_diff -= 3; } if (cur_diff < best_diff) { best_diff = cur_diff; - best_tx = tx; + best_tx.green_to_red_ = green_to_red; } } best_diff = MAX_DIFF_COST; - green_to_red = best_tx.green_to_red_; for (green_to_blue = -32; green_to_blue <= 32; green_to_blue += step) { for (red_to_blue = -32; red_to_blue <= 32; red_to_blue += step) { int all_y; int histo[256] = { 0 }; - Multipliers tx; - tx.green_to_red_ = green_to_red; - tx.green_to_blue_ = green_to_blue; - tx.red_to_blue_ = red_to_blue; for (all_y = tile_y_offset; all_y < all_y_max; ++all_y) { - uint32_t predict; int all_x; int ix = all_y * xsize + tile_x_offset; for (all_x = tile_x_offset; all_x < all_x_max; ++all_x, ++ix) { if (SkipRepeatedPixels(argb, ix, xsize)) { continue; } - predict = TransformColor(&tx, argb[ix], 0); - ++histo[predict & 0xff]; // blue. + ++histo[TransformColorBlue(green_to_blue, red_to_blue, argb[ix])]; } } cur_diff = - PredictionCostCrossColor(&accumulated_blue_histo[0], &histo[0]); - if (tx.green_to_blue_ == prevX.green_to_blue_) { + PredictionCostCrossColor(&accumulated_blue_histo[0], &histo[0]); + if ((uint8_t)green_to_blue == prevX.green_to_blue_) { cur_diff -= 3; // favor keeping the areas locally similar } - if (tx.green_to_blue_ == prevY.green_to_blue_) { + if ((uint8_t)green_to_blue == prevY.green_to_blue_) { cur_diff -= 3; // favor keeping the areas locally similar } - if (tx.red_to_blue_ == prevX.red_to_blue_) { + if ((uint8_t)red_to_blue == prevX.red_to_blue_) { cur_diff -= 3; // favor keeping the areas locally similar } - if (tx.red_to_blue_ == prevY.red_to_blue_) { + if ((uint8_t)red_to_blue == prevY.red_to_blue_) { cur_diff -= 3; // favor keeping the areas locally similar } - if (tx.green_to_blue_ == 0) { + if (green_to_blue == 0) { cur_diff -= 3; } - if (tx.red_to_blue_ == 0) { + if (red_to_blue == 0) { cur_diff -= 3; } if (cur_diff < best_diff) { best_diff = cur_diff; - best_tx = tx; + best_tx.green_to_blue_ = green_to_blue; + best_tx.red_to_blue_ = red_to_blue; } } } @@ -935,7 +1108,7 @@ static void ColorIndexInverseTransform( uint32_t packed_pixels = 0; int x; for (x = 0; x < width; ++x) { - // We need to load fresh 'packed_pixels' once every 'bytes_per_pixels' + // We need to load fresh 'packed_pixels' once every 'pixels_per_byte' // increments of x. Fortunately, pixels_per_byte is a power of 2, so // can just use a mask for that, instead of decrementing a counter. if ((x & count_mask) == 0) packed_pixels = ((*src++) >> 8) & 0xff; @@ -976,7 +1149,21 @@ void VP8LInverseTransform(const VP8LTransform* const transform, ColorSpaceInverseTransform(transform, row_start, row_end, out); break; case COLOR_INDEXING_TRANSFORM: - ColorIndexInverseTransform(transform, row_start, row_end, in, out); + if (in == out && transform->bits_ > 0) { + // Move packed pixels to the end of unpacked region, so that unpacking + // can occur seamlessly. + // Also, note that this is the only transform that applies on + // the effective width of VP8LSubSampleSize(xsize_, bits_). All other + // transforms work on effective width of xsize_. + const int out_stride = (row_end - row_start) * transform->xsize_; + const int in_stride = (row_end - row_start) * + VP8LSubSampleSize(transform->xsize_, transform->bits_); + uint32_t* const src = out + out_stride - in_stride; + memmove(src, out, in_stride * sizeof(*src)); + ColorIndexInverseTransform(transform, row_start, row_end, src, out); + } else { + ColorIndexInverseTransform(transform, row_start, row_end, in, out); + } break; } } @@ -1020,8 +1207,15 @@ static void ConvertBGRAToRGBA4444(const uint32_t* src, const uint32_t* const src_end = src + num_pixels; while (src < src_end) { const uint32_t argb = *src++; - *dst++ = ((argb >> 16) & 0xf0) | ((argb >> 12) & 0xf); - *dst++ = ((argb >> 0) & 0xf0) | ((argb >> 28) & 0xf); + const uint8_t rg = ((argb >> 16) & 0xf0) | ((argb >> 12) & 0xf); + const uint8_t ba = ((argb >> 0) & 0xf0) | ((argb >> 28) & 0xf); +#ifdef WEBP_SWAP_16BIT_CSP + *dst++ = ba; + *dst++ = rg; +#else + *dst++ = rg; + *dst++ = ba; +#endif } } @@ -1030,8 +1224,15 @@ static void ConvertBGRAToRGB565(const uint32_t* src, const uint32_t* const src_end = src + num_pixels; while (src < src_end) { const uint32_t argb = *src++; - *dst++ = ((argb >> 16) & 0xf8) | ((argb >> 13) & 0x7); - *dst++ = ((argb >> 5) & 0xe0) | ((argb >> 3) & 0x1f); + const uint8_t rg = ((argb >> 16) & 0xf8) | ((argb >> 13) & 0x7); + const uint8_t gb = ((argb >> 5) & 0xe0) | ((argb >> 3) & 0x1f); +#ifdef WEBP_SWAP_16BIT_CSP + *dst++ = gb; + *dst++ = rg; +#else + *dst++ = rg; + *dst++ = gb; +#endif } } @@ -1052,20 +1253,27 @@ static void CopyOrSwap(const uint32_t* src, int num_pixels, uint8_t* dst, const uint32_t* const src_end = src + num_pixels; while (src < src_end) { uint32_t argb = *src++; + +#if !defined(WEBP_REFERENCE_IMPLEMENTATION) #if !defined(__BIG_ENDIAN__) && (defined(__i386__) || defined(__x86_64__)) __asm__ volatile("bswap %0" : "=r"(argb) : "0"(argb)); *(uint32_t*)dst = argb; - dst += sizeof(argb); #elif !defined(__BIG_ENDIAN__) && defined(_MSC_VER) argb = _byteswap_ulong(argb); *(uint32_t*)dst = argb; - dst += sizeof(argb); #else - *dst++ = (argb >> 24) & 0xff; - *dst++ = (argb >> 16) & 0xff; - *dst++ = (argb >> 8) & 0xff; - *dst++ = (argb >> 0) & 0xff; + dst[0] = (argb >> 24) & 0xff; + dst[1] = (argb >> 16) & 0xff; + dst[2] = (argb >> 8) & 0xff; + dst[3] = (argb >> 0) & 0xff; #endif +#else // WEBP_REFERENCE_IMPLEMENTATION + dst[0] = (argb >> 24) & 0xff; + dst[1] = (argb >> 16) & 0xff; + dst[2] = (argb >> 8) & 0xff; + dst[3] = (argb >> 0) & 0xff; +#endif + dst += sizeof(argb); } } else { memcpy(dst, src, num_pixels * sizeof(*src)); diff --git a/third_party/libwebp/dsp/lossless.h b/third_party/libwebp/dsp/lossless.h index 992516f..0ac4ecb 100644 --- a/third_party/libwebp/dsp/lossless.h +++ b/third_party/libwebp/dsp/lossless.h @@ -59,10 +59,20 @@ static WEBP_INLINE uint32_t VP8LSubSampleSize(uint32_t size, return (size + (1 << sampling_bits) - 1) >> sampling_bits; } -// Faster logarithm for integers, with the property of log2(0) == 0. -float VP8LFastLog2(int v); +// Faster logarithm for integers. Small values use a look-up table. +#define LOG_LOOKUP_IDX_MAX 256 +extern const float kLog2Table[LOG_LOOKUP_IDX_MAX]; +extern const float kSLog2Table[LOG_LOOKUP_IDX_MAX]; +extern float VP8LFastLog2Slow(int v); +extern float VP8LFastSLog2Slow(int v); +static WEBP_INLINE float VP8LFastLog2(int v) { + return (v < LOG_LOOKUP_IDX_MAX) ? kLog2Table[v] : VP8LFastLog2Slow(v); +} // Fast calculation of v * log2(v) for integer input. -static WEBP_INLINE float VP8LFastSLog2(int v) { return VP8LFastLog2(v) * v; } +static WEBP_INLINE float VP8LFastSLog2(int v) { + return (v < LOG_LOOKUP_IDX_MAX) ? kSLog2Table[v] : VP8LFastSLog2Slow(v); +} + // In-place difference of each component with mod 256. static WEBP_INLINE uint32_t VP8LSubPixels(uint32_t a, uint32_t b) { diff --git a/third_party/libwebp/dsp/upsampling.c b/third_party/libwebp/dsp/upsampling.c index 9ca0492..91d939c 100644 --- a/third_party/libwebp/dsp/upsampling.c +++ b/third_party/libwebp/dsp/upsampling.c @@ -271,8 +271,7 @@ static void ApplyAlphaMultiply(uint8_t* rgba, int alpha_first, // rgbA4444 -#define MULTIPLIER(a) ((a) * 0x11) -#define PREMULTIPLY(x, m) (((x) * (m)) >> 12) +#define MULTIPLIER(a) ((a) * 0x1111) // 0x1111 ~= (1 << 16) / 15 static WEBP_INLINE uint8_t dither_hi(uint8_t x) { return (x & 0xf0) | (x >> 4); @@ -282,24 +281,27 @@ static WEBP_INLINE uint8_t dither_lo(uint8_t x) { return (x & 0x0f) | (x << 4); } +static WEBP_INLINE uint8_t multiply(uint8_t x, uint32_t m) { + return (x * m) >> 16; +} + static void ApplyAlphaMultiply4444(uint8_t* rgba4444, int w, int h, int stride) { while (h-- > 0) { int i; for (i = 0; i < w; ++i) { - const uint8_t a = dither_lo(rgba4444[2 * i + 1]); + const uint8_t a = (rgba4444[2 * i + 1] & 0x0f); const uint32_t mult = MULTIPLIER(a); - const uint8_t r = PREMULTIPLY(dither_hi(rgba4444[2 * i + 0]), mult); - const uint8_t g = PREMULTIPLY(dither_lo(rgba4444[2 * i + 0]), mult); - const uint8_t b = PREMULTIPLY(dither_hi(rgba4444[2 * i + 1]), mult); - rgba4444[2 * i + 0] = (r & 0xf0) | (g & 0x0f); + const uint8_t r = multiply(dither_hi(rgba4444[2 * i + 0]), mult); + const uint8_t g = multiply(dither_lo(rgba4444[2 * i + 0]), mult); + const uint8_t b = multiply(dither_hi(rgba4444[2 * i + 1]), mult); + rgba4444[2 * i + 0] = (r & 0xf0) | ((g >> 4) & 0x0f); rgba4444[2 * i + 1] = (b & 0xf0) | a; } rgba4444 += stride; } } #undef MULTIPLIER -#undef PREMULTIPLY void (*WebPApplyAlphaMultiply)(uint8_t*, int, int, int, int) = ApplyAlphaMultiply; @@ -326,6 +328,11 @@ void WebPInitUpsamplers(void) { WebPInitUpsamplersSSE2(); } #endif +#if defined(WEBP_USE_NEON) + if (VP8GetCPUInfo(kNEON)) { + WebPInitUpsamplersNEON(); + } +#endif } #endif // FANCY_UPSAMPLING } @@ -346,6 +353,11 @@ void WebPInitPremultiply(void) { WebPInitPremultiplySSE2(); } #endif +#if defined(WEBP_USE_NEON) + if (VP8GetCPUInfo(kNEON)) { + WebPInitPremultiplyNEON(); + } +#endif } #endif // FANCY_UPSAMPLING } diff --git a/third_party/libwebp/dsp/upsampling_neon.c b/third_party/libwebp/dsp/upsampling_neon.c new file mode 100644 index 0000000..00e2f89 --- /dev/null +++ b/third_party/libwebp/dsp/upsampling_neon.c @@ -0,0 +1,292 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// This code is licensed under the same terms as WebM: +// Software License Agreement: http://www.webmproject.org/license/software/ +// Additional IP Rights Grant: http://www.webmproject.org/license/additional/ +// ----------------------------------------------------------------------------- +// +// NEON version of YUV to RGB upsampling functions. +// +// Author: mans@mansr.com (Mans Rullgard) +// Based on SSE code by: somnath@google.com (Somnath Banerjee) + +#include "./dsp.h" + +#if defined(__cplusplus) || defined(c_plusplus) +extern "C" { +#endif + +#if defined(WEBP_USE_NEON) + +#include +#include +#include +#include "./yuv.h" + +#ifdef FANCY_UPSAMPLING + +// Loads 9 pixels each from rows r1 and r2 and generates 16 pixels. +#define UPSAMPLE_16PIXELS(r1, r2, out) { \ + uint8x8_t a = vld1_u8(r1); \ + uint8x8_t b = vld1_u8(r1 + 1); \ + uint8x8_t c = vld1_u8(r2); \ + uint8x8_t d = vld1_u8(r2 + 1); \ + \ + uint16x8_t al = vshll_n_u8(a, 1); \ + uint16x8_t bl = vshll_n_u8(b, 1); \ + uint16x8_t cl = vshll_n_u8(c, 1); \ + uint16x8_t dl = vshll_n_u8(d, 1); \ + \ + uint8x8_t diag1, diag2; \ + uint16x8_t sl; \ + \ + /* a + b + c + d */ \ + sl = vaddl_u8(a, b); \ + sl = vaddw_u8(sl, c); \ + sl = vaddw_u8(sl, d); \ + \ + al = vaddq_u16(sl, al); /* 3a + b + c + d */ \ + bl = vaddq_u16(sl, bl); /* a + 3b + c + d */ \ + \ + al = vaddq_u16(al, dl); /* 3a + b + c + 3d */ \ + bl = vaddq_u16(bl, cl); /* a + 3b + 3c + d */ \ + \ + diag2 = vshrn_n_u16(al, 3); \ + diag1 = vshrn_n_u16(bl, 3); \ + \ + a = vrhadd_u8(a, diag1); \ + b = vrhadd_u8(b, diag2); \ + c = vrhadd_u8(c, diag2); \ + d = vrhadd_u8(d, diag1); \ + \ + { \ + const uint8x8x2_t a_b = {{ a, b }}; \ + const uint8x8x2_t c_d = {{ c, d }}; \ + vst2_u8(out, a_b); \ + vst2_u8(out + 32, c_d); \ + } \ +} + +// Turn the macro into a function for reducing code-size when non-critical +static void Upsample16Pixels(const uint8_t *r1, const uint8_t *r2, + uint8_t *out) { + UPSAMPLE_16PIXELS(r1, r2, out); +} + +#define UPSAMPLE_LAST_BLOCK(tb, bb, num_pixels, out) { \ + uint8_t r1[9], r2[9]; \ + memcpy(r1, (tb), (num_pixels)); \ + memcpy(r2, (bb), (num_pixels)); \ + /* replicate last byte */ \ + memset(r1 + (num_pixels), r1[(num_pixels) - 1], 9 - (num_pixels)); \ + memset(r2 + (num_pixels), r2[(num_pixels) - 1], 9 - (num_pixels)); \ + Upsample16Pixels(r1, r2, out); \ +} + +#define CY 76283 +#define CVR 89858 +#define CUG 22014 +#define CVG 45773 +#define CUB 113618 + +static const int16_t coef[4] = { CVR / 4, CUG, CVG / 2, CUB / 4 }; + +#define CONVERT8(FMT, XSTEP, N, src_y, src_uv, out, cur_x) { \ + int i; \ + for (i = 0; i < N; i += 8) { \ + int off = ((cur_x) + i) * XSTEP; \ + uint8x8_t y = vld1_u8(src_y + (cur_x) + i); \ + uint8x8_t u = vld1_u8((src_uv) + i); \ + uint8x8_t v = vld1_u8((src_uv) + i + 16); \ + int16x8_t yy = vreinterpretq_s16_u16(vsubl_u8(y, u16)); \ + int16x8_t uu = vreinterpretq_s16_u16(vsubl_u8(u, u128)); \ + int16x8_t vv = vreinterpretq_s16_u16(vsubl_u8(v, u128)); \ + \ + int16x8_t ud = vshlq_n_s16(uu, 1); \ + int16x8_t vd = vshlq_n_s16(vv, 1); \ + \ + int32x4_t vrl = vqdmlal_lane_s16(vshll_n_s16(vget_low_s16(vv), 1), \ + vget_low_s16(vd), cf16, 0); \ + int32x4_t vrh = vqdmlal_lane_s16(vshll_n_s16(vget_high_s16(vv), 1), \ + vget_high_s16(vd), cf16, 0); \ + int16x8_t vr = vcombine_s16(vrshrn_n_s32(vrl, 16), \ + vrshrn_n_s32(vrh, 16)); \ + \ + int32x4_t vl = vmovl_s16(vget_low_s16(vv)); \ + int32x4_t vh = vmovl_s16(vget_high_s16(vv)); \ + int32x4_t ugl = vmlal_lane_s16(vl, vget_low_s16(uu), cf16, 1); \ + int32x4_t ugh = vmlal_lane_s16(vh, vget_high_s16(uu), cf16, 1); \ + int32x4_t gcl = vqdmlal_lane_s16(ugl, vget_low_s16(vv), cf16, 2); \ + int32x4_t gch = vqdmlal_lane_s16(ugh, vget_high_s16(vv), cf16, 2); \ + int16x8_t gc = vcombine_s16(vrshrn_n_s32(gcl, 16), \ + vrshrn_n_s32(gch, 16)); \ + \ + int32x4_t ubl = vqdmlal_lane_s16(vshll_n_s16(vget_low_s16(uu), 1), \ + vget_low_s16(ud), cf16, 3); \ + int32x4_t ubh = vqdmlal_lane_s16(vshll_n_s16(vget_high_s16(uu), 1), \ + vget_high_s16(ud), cf16, 3); \ + int16x8_t ub = vcombine_s16(vrshrn_n_s32(ubl, 16), \ + vrshrn_n_s32(ubh, 16)); \ + \ + int32x4_t rl = vaddl_s16(vget_low_s16(yy), vget_low_s16(vr)); \ + int32x4_t rh = vaddl_s16(vget_high_s16(yy), vget_high_s16(vr)); \ + int32x4_t gl = vsubl_s16(vget_low_s16(yy), vget_low_s16(gc)); \ + int32x4_t gh = vsubl_s16(vget_high_s16(yy), vget_high_s16(gc)); \ + int32x4_t bl = vaddl_s16(vget_low_s16(yy), vget_low_s16(ub)); \ + int32x4_t bh = vaddl_s16(vget_high_s16(yy), vget_high_s16(ub)); \ + \ + rl = vmulq_lane_s32(rl, cf32, 0); \ + rh = vmulq_lane_s32(rh, cf32, 0); \ + gl = vmulq_lane_s32(gl, cf32, 0); \ + gh = vmulq_lane_s32(gh, cf32, 0); \ + bl = vmulq_lane_s32(bl, cf32, 0); \ + bh = vmulq_lane_s32(bh, cf32, 0); \ + \ + y = vqmovun_s16(vcombine_s16(vrshrn_n_s32(rl, 16), \ + vrshrn_n_s32(rh, 16))); \ + u = vqmovun_s16(vcombine_s16(vrshrn_n_s32(gl, 16), \ + vrshrn_n_s32(gh, 16))); \ + v = vqmovun_s16(vcombine_s16(vrshrn_n_s32(bl, 16), \ + vrshrn_n_s32(bh, 16))); \ + STR_ ## FMT(out + off, y, u, v); \ + } \ +} + +#define v255 vmov_n_u8(255) + +#define STR_Rgb(out, r, g, b) do { \ + const uint8x8x3_t r_g_b = {{ r, g, b }}; \ + vst3_u8(out, r_g_b); \ +} while (0) + +#define STR_Bgr(out, r, g, b) do { \ + const uint8x8x3_t b_g_r = {{ b, g, r }}; \ + vst3_u8(out, b_g_r); \ +} while (0) + +#define STR_Rgba(out, r, g, b) do { \ + const uint8x8x4_t r_g_b_v255 = {{ r, g, b, v255 }}; \ + vst4_u8(out, r_g_b_v255); \ +} while (0) + +#define STR_Bgra(out, r, g, b) do { \ + const uint8x8x4_t b_g_r_v255 = {{ b, g, r, v255 }}; \ + vst4_u8(out, b_g_r_v255); \ +} while (0) + +#define CONVERT1(FMT, XSTEP, N, src_y, src_uv, rgb, cur_x) { \ + int i; \ + for (i = 0; i < N; i++) { \ + int off = ((cur_x) + i) * XSTEP; \ + int y = src_y[(cur_x) + i]; \ + int u = (src_uv)[i]; \ + int v = (src_uv)[i + 16]; \ + VP8YuvTo ## FMT(y, u, v, rgb + off); \ + } \ +} + +#define CONVERT2RGB_8(FMT, XSTEP, top_y, bottom_y, uv, \ + top_dst, bottom_dst, cur_x, len) { \ + if (top_y) { \ + CONVERT8(FMT, XSTEP, len, top_y, uv, top_dst, cur_x) \ + } \ + if (bottom_y) { \ + CONVERT8(FMT, XSTEP, len, bottom_y, (uv) + 32, bottom_dst, cur_x) \ + } \ +} + +#define CONVERT2RGB_1(FMT, XSTEP, top_y, bottom_y, uv, \ + top_dst, bottom_dst, cur_x, len) { \ + if (top_y) { \ + CONVERT1(FMT, XSTEP, len, top_y, uv, top_dst, cur_x); \ + } \ + if (bottom_y) { \ + CONVERT1(FMT, XSTEP, len, bottom_y, (uv) + 32, bottom_dst, cur_x); \ + } \ +} + +#define NEON_UPSAMPLE_FUNC(FUNC_NAME, FMT, XSTEP) \ +static void FUNC_NAME(const uint8_t *top_y, const uint8_t *bottom_y, \ + const uint8_t *top_u, const uint8_t *top_v, \ + const uint8_t *cur_u, const uint8_t *cur_v, \ + uint8_t *top_dst, uint8_t *bottom_dst, int len) { \ + int block; \ + /* 16 byte aligned array to cache reconstructed u and v */ \ + uint8_t uv_buf[2 * 32 + 15]; \ + uint8_t *const r_uv = (uint8_t*)((uintptr_t)(uv_buf + 15) & ~15); \ + const int uv_len = (len + 1) >> 1; \ + /* 9 pixels must be read-able for each block */ \ + const int num_blocks = (uv_len - 1) >> 3; \ + const int leftover = uv_len - num_blocks * 8; \ + const int last_pos = 1 + 16 * num_blocks; \ + \ + const int u_diag = ((top_u[0] + cur_u[0]) >> 1) + 1; \ + const int v_diag = ((top_v[0] + cur_v[0]) >> 1) + 1; \ + \ + const int16x4_t cf16 = vld1_s16(coef); \ + const int32x2_t cf32 = vmov_n_s32(CY); \ + const uint8x8_t u16 = vmov_n_u8(16); \ + const uint8x8_t u128 = vmov_n_u8(128); \ + \ + /* Treat the first pixel in regular way */ \ + if (top_y) { \ + const int u0 = (top_u[0] + u_diag) >> 1; \ + const int v0 = (top_v[0] + v_diag) >> 1; \ + VP8YuvTo ## FMT(top_y[0], u0, v0, top_dst); \ + } \ + if (bottom_y) { \ + const int u0 = (cur_u[0] + u_diag) >> 1; \ + const int v0 = (cur_v[0] + v_diag) >> 1; \ + VP8YuvTo ## FMT(bottom_y[0], u0, v0, bottom_dst); \ + } \ + \ + for (block = 0; block < num_blocks; ++block) { \ + UPSAMPLE_16PIXELS(top_u, cur_u, r_uv); \ + UPSAMPLE_16PIXELS(top_v, cur_v, r_uv + 16); \ + CONVERT2RGB_8(FMT, XSTEP, top_y, bottom_y, r_uv, \ + top_dst, bottom_dst, 16 * block + 1, 16); \ + top_u += 8; \ + cur_u += 8; \ + top_v += 8; \ + cur_v += 8; \ + } \ + \ + UPSAMPLE_LAST_BLOCK(top_u, cur_u, leftover, r_uv); \ + UPSAMPLE_LAST_BLOCK(top_v, cur_v, leftover, r_uv + 16); \ + CONVERT2RGB_1(FMT, XSTEP, top_y, bottom_y, r_uv, \ + top_dst, bottom_dst, last_pos, len - last_pos); \ +} + +// NEON variants of the fancy upsampler. +NEON_UPSAMPLE_FUNC(UpsampleRgbLinePairNEON, Rgb, 3) +NEON_UPSAMPLE_FUNC(UpsampleBgrLinePairNEON, Bgr, 3) +NEON_UPSAMPLE_FUNC(UpsampleRgbaLinePairNEON, Rgba, 4) +NEON_UPSAMPLE_FUNC(UpsampleBgraLinePairNEON, Bgra, 4) + +#endif // FANCY_UPSAMPLING + +#endif // WEBP_USE_NEON + +//------------------------------------------------------------------------------ + +extern WebPUpsampleLinePairFunc WebPUpsamplers[/* MODE_LAST */]; + +void WebPInitUpsamplersNEON(void) { +#if defined(WEBP_USE_NEON) + WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePairNEON; + WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePairNEON; + WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePairNEON; + WebPUpsamplers[MODE_BGRA] = UpsampleBgraLinePairNEON; +#endif // WEBP_USE_NEON +} + +void WebPInitPremultiplyNEON(void) { +#if defined(WEBP_USE_NEON) + WebPUpsamplers[MODE_rgbA] = UpsampleRgbaLinePairNEON; + WebPUpsamplers[MODE_bgrA] = UpsampleBgraLinePairNEON; +#endif // WEBP_USE_NEON +} + +#if defined(__cplusplus) || defined(c_plusplus) +} // extern "C" +#endif diff --git a/third_party/libwebp/dsp/upsampling_sse2.c b/third_party/libwebp/dsp/upsampling_sse2.c index 8cb275a..ba075d1 100644 --- a/third_party/libwebp/dsp/upsampling_sse2.c +++ b/third_party/libwebp/dsp/upsampling_sse2.c @@ -11,6 +11,10 @@ #include "./dsp.h" +#if defined(__cplusplus) || defined(c_plusplus) +extern "C" { +#endif + #if defined(WEBP_USE_SSE2) #include @@ -18,10 +22,6 @@ #include #include "./yuv.h" -#if defined(__cplusplus) || defined(c_plusplus) -extern "C" { -#endif - #ifdef FANCY_UPSAMPLING // We compute (9*a + 3*b + 3*c + d + 8) / 16 as follows @@ -51,12 +51,12 @@ extern "C" { // pack and store two alterning pixel rows #define PACK_AND_STORE(a, b, da, db, out) do { \ - const __m128i ta = _mm_avg_epu8(a, da); /* (9a + 3b + 3c + d + 8) / 16 */ \ - const __m128i tb = _mm_avg_epu8(b, db); /* (3a + 9b + c + 3d + 8) / 16 */ \ - const __m128i t1 = _mm_unpacklo_epi8(ta, tb); \ - const __m128i t2 = _mm_unpackhi_epi8(ta, tb); \ - _mm_store_si128(((__m128i*)(out)) + 0, t1); \ - _mm_store_si128(((__m128i*)(out)) + 1, t2); \ + const __m128i t_a = _mm_avg_epu8(a, da); /* (9a + 3b + 3c + d + 8) / 16 */ \ + const __m128i t_b = _mm_avg_epu8(b, db); /* (3a + 9b + c + 3d + 8) / 16 */ \ + const __m128i t_1 = _mm_unpacklo_epi8(t_a, t_b); \ + const __m128i t_2 = _mm_unpackhi_epi8(t_a, t_b); \ + _mm_store_si128(((__m128i*)(out)) + 0, t_1); \ + _mm_store_si128(((__m128i*)(out)) + 1, t_2); \ } while (0) // Loads 17 pixels each from rows r1 and r2 and generates 32 pixels. @@ -128,7 +128,7 @@ static void FUNC_NAME(const uint8_t* top_y, const uint8_t* bottom_y, \ const uint8_t* top_u, const uint8_t* top_v, \ const uint8_t* cur_u, const uint8_t* cur_v, \ uint8_t* top_dst, uint8_t* bottom_dst, int len) { \ - int b; \ + int block; \ /* 16 byte aligned array to cache reconstructed u and v */ \ uint8_t uv_buf[4 * 32 + 15]; \ uint8_t* const r_uv = (uint8_t*)((uintptr_t)(uv_buf + 15) & ~15); \ @@ -154,11 +154,11 @@ static void FUNC_NAME(const uint8_t* top_y, const uint8_t* bottom_y, \ FUNC(bottom_y[0], u0, v0, bottom_dst); \ } \ \ - for (b = 0; b < num_blocks; ++b) { \ + for (block = 0; block < num_blocks; ++block) { \ UPSAMPLE_32PIXELS(top_u, cur_u, r_uv + 0 * 32); \ UPSAMPLE_32PIXELS(top_v, cur_v, r_uv + 1 * 32); \ CONVERT2RGB(FUNC, XSTEP, top_y, bottom_y, r_uv, top_dst, bottom_dst, \ - 32 * b + 1, 32) \ + 32 * block + 1, 32) \ top_u += 16; \ cur_u += 16; \ top_v += 16; \ @@ -184,26 +184,32 @@ SSE2_UPSAMPLE_FUNC(UpsampleBgraLinePairSSE2, VP8YuvToBgra, 4) #undef CONVERT2RGB #undef SSE2_UPSAMPLE_FUNC +#endif // FANCY_UPSAMPLING + +#endif // WEBP_USE_SSE2 + //------------------------------------------------------------------------------ extern WebPUpsampleLinePairFunc WebPUpsamplers[/* MODE_LAST */]; void WebPInitUpsamplersSSE2(void) { +#if defined(WEBP_USE_SSE2) WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePairSSE2; WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePairSSE2; WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePairSSE2; WebPUpsamplers[MODE_BGRA] = UpsampleBgraLinePairSSE2; +#endif // WEBP_USE_SSE2 } void WebPInitPremultiplySSE2(void) { +#if defined(WEBP_USE_SSE2) WebPUpsamplers[MODE_rgbA] = UpsampleRgbaLinePairSSE2; WebPUpsamplers[MODE_bgrA] = UpsampleBgraLinePairSSE2; +#endif // WEBP_USE_SSE2 } -#endif // FANCY_UPSAMPLING - #if defined(__cplusplus) || defined(c_plusplus) } // extern "C" #endif -#endif // WEBP_USE_SSE2 + diff --git a/third_party/libwebp/dsp/yuv.c b/third_party/libwebp/dsp/yuv.c index 7f05f9a..f8988ba 100644 --- a/third_party/libwebp/dsp/yuv.c +++ b/third_party/libwebp/dsp/yuv.c @@ -15,7 +15,7 @@ extern "C" { #endif -enum { YUV_HALF = 1 << (YUV_FIX - 1) }; +#ifdef WEBP_YUV_USE_TABLE int16_t VP8kVToR[256], VP8kUToB[256]; int32_t VP8kVToG[256], VP8kUToG[256]; @@ -33,6 +33,7 @@ void VP8YUVInit(void) { if (done) { return; } +#ifndef USE_YUVj for (i = 0; i < 256; ++i) { VP8kVToR[i] = (89858 * (i - 128) + YUV_HALF) >> YUV_FIX; VP8kUToG[i] = -22014 * (i - 128) + YUV_HALF; @@ -44,9 +45,29 @@ void VP8YUVInit(void) { VP8kClip[i - YUV_RANGE_MIN] = clip(k, 255); VP8kClip4Bits[i - YUV_RANGE_MIN] = clip((k + 8) >> 4, 15); } +#else + for (i = 0; i < 256; ++i) { + VP8kVToR[i] = (91881 * (i - 128) + YUV_HALF) >> YUV_FIX; + VP8kUToG[i] = -22554 * (i - 128) + YUV_HALF; + VP8kVToG[i] = -46802 * (i - 128); + VP8kUToB[i] = (116130 * (i - 128) + YUV_HALF) >> YUV_FIX; + } + for (i = YUV_RANGE_MIN; i < YUV_RANGE_MAX; ++i) { + const int k = i; + VP8kClip[i - YUV_RANGE_MIN] = clip(k, 255); + VP8kClip4Bits[i - YUV_RANGE_MIN] = clip((k + 8) >> 4, 15); + } +#endif + done = 1; } +#else + +void VP8YUVInit(void) {} + +#endif // WEBP_YUV_USE_TABLE + #if defined(__cplusplus) || defined(c_plusplus) } // extern "C" #endif diff --git a/third_party/libwebp/dsp/yuv.h b/third_party/libwebp/dsp/yuv.h index 22cb259..126404b 100644 --- a/third_party/libwebp/dsp/yuv.h +++ b/third_party/libwebp/dsp/yuv.h @@ -7,6 +7,26 @@ // // inline YUV<->RGB conversion function // +// The exact naming is Y'CbCr, following the ITU-R BT.601 standard. +// More information at: http://en.wikipedia.org/wiki/YCbCr +// Y = 0.2569 * R + 0.5044 * G + 0.0979 * B + 16 +// U = -0.1483 * R - 0.2911 * G + 0.4394 * B + 128 +// V = 0.4394 * R - 0.3679 * G - 0.0715 * B + 128 +// We use 16bit fixed point operations for RGB->YUV conversion. +// +// For the Y'CbCr to RGB conversion, the BT.601 specification reads: +// R = 1.164 * (Y-16) + 1.596 * (V-128) +// G = 1.164 * (Y-16) - 0.813 * (V-128) - 0.391 * (U-128) +// B = 1.164 * (Y-16) + 2.018 * (U-128) +// where Y is in the [16,235] range, and U/V in the [16,240] range. +// In the table-lookup version (WEBP_YUV_USE_TABLE), the common factor +// "1.164 * (Y-16)" can be handled as an offset in the VP8kClip[] table. +// So in this case the formulae should be read as: +// R = 1.164 * [Y + 1.371 * (V-128) ] - 18.624 +// G = 1.164 * [Y - 0.698 * (V-128) - 0.336 * (U-128)] - 18.624 +// B = 1.164 * [Y + 1.733 * (U-128)] - 18.624 +// once factorized. Here too, 16bit fixed precision is used. +// // Author: Skal (pascal.massimino@gmail.com) #ifndef WEBP_DSP_YUV_H_ @@ -14,6 +34,19 @@ #include "../dec/decode_vp8.h" +// Define the following to use the LUT-based code: +#define WEBP_YUV_USE_TABLE + +#if defined(WEBP_EXPERIMENTAL_FEATURES) +// Do NOT activate this feature for real compression. This is only experimental! +// This flag is for comparison purpose against JPEG's "YUVj" natural colorspace. +// This colorspace is close to Rec.601's Y'CbCr model with the notable +// difference of allowing larger range for luma/chroma. +// See http://en.wikipedia.org/wiki/YCbCr#JPEG_conversion paragraph, and its +// difference with http://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion +// #define USE_YUVj +#endif + //------------------------------------------------------------------------------ // YUV -> RGB conversion @@ -22,9 +55,14 @@ extern "C" { #endif enum { YUV_FIX = 16, // fixed-point precision + YUV_HALF = 1 << (YUV_FIX - 1), + YUV_MASK = (256 << YUV_FIX) - 1, YUV_RANGE_MIN = -227, // min value of r/g/b output YUV_RANGE_MAX = 256 + 226 // max value of r/g/b output }; + +#ifdef WEBP_YUV_USE_TABLE + extern int16_t VP8kVToR[256], VP8kUToB[256]; extern int32_t VP8kVToG[256], VP8kUToG[256]; extern uint8_t VP8kClip[YUV_RANGE_MAX - YUV_RANGE_MIN]; @@ -40,21 +78,32 @@ static WEBP_INLINE void VP8YuvToRgb(uint8_t y, uint8_t u, uint8_t v, rgb[2] = VP8kClip[y + b_off - YUV_RANGE_MIN]; } -static WEBP_INLINE void VP8YuvToRgb565(uint8_t y, uint8_t u, uint8_t v, - uint8_t* const rgb) { +static WEBP_INLINE void VP8YuvToBgr(uint8_t y, uint8_t u, uint8_t v, + uint8_t* const bgr) { const int r_off = VP8kVToR[v]; const int g_off = (VP8kVToG[v] + VP8kUToG[u]) >> YUV_FIX; const int b_off = VP8kUToB[u]; - rgb[0] = ((VP8kClip[y + r_off - YUV_RANGE_MIN] & 0xf8) | - (VP8kClip[y + g_off - YUV_RANGE_MIN] >> 5)); - rgb[1] = (((VP8kClip[y + g_off - YUV_RANGE_MIN] << 3) & 0xe0) | - (VP8kClip[y + b_off - YUV_RANGE_MIN] >> 3)); + bgr[0] = VP8kClip[y + b_off - YUV_RANGE_MIN]; + bgr[1] = VP8kClip[y + g_off - YUV_RANGE_MIN]; + bgr[2] = VP8kClip[y + r_off - YUV_RANGE_MIN]; } -static WEBP_INLINE void VP8YuvToArgb(uint8_t y, uint8_t u, uint8_t v, - uint8_t* const argb) { - argb[0] = 0xff; - VP8YuvToRgb(y, u, v, argb + 1); +static WEBP_INLINE void VP8YuvToRgb565(uint8_t y, uint8_t u, uint8_t v, + uint8_t* const rgb) { + const int r_off = VP8kVToR[v]; + const int g_off = (VP8kVToG[v] + VP8kUToG[u]) >> YUV_FIX; + const int b_off = VP8kUToB[u]; + const uint8_t rg = ((VP8kClip[y + r_off - YUV_RANGE_MIN] & 0xf8) | + (VP8kClip[y + g_off - YUV_RANGE_MIN] >> 5)); + const uint8_t gb = (((VP8kClip[y + g_off - YUV_RANGE_MIN] << 3) & 0xe0) | + (VP8kClip[y + b_off - YUV_RANGE_MIN] >> 3)); +#ifdef WEBP_SWAP_16BIT_CSP + rgb[0] = gb; + rgb[1] = rg; +#else + rgb[0] = rg; + rgb[1] = gb; +#endif } static WEBP_INLINE void VP8YuvToRgba4444(uint8_t y, uint8_t u, uint8_t v, @@ -62,20 +111,104 @@ static WEBP_INLINE void VP8YuvToRgba4444(uint8_t y, uint8_t u, uint8_t v, const int r_off = VP8kVToR[v]; const int g_off = (VP8kVToG[v] + VP8kUToG[u]) >> YUV_FIX; const int b_off = VP8kUToB[u]; - // Don't update alpha (last 4 bits of argb[1]) - argb[0] = ((VP8kClip4Bits[y + r_off - YUV_RANGE_MIN] << 4) | - VP8kClip4Bits[y + g_off - YUV_RANGE_MIN]); - argb[1] = 0x0f | (VP8kClip4Bits[y + b_off - YUV_RANGE_MIN] << 4); + const uint8_t rg = ((VP8kClip4Bits[y + r_off - YUV_RANGE_MIN] << 4) | + VP8kClip4Bits[y + g_off - YUV_RANGE_MIN]); + const uint8_t ba = (VP8kClip4Bits[y + b_off - YUV_RANGE_MIN] << 4) | 0x0f; +#ifdef WEBP_SWAP_16BIT_CSP + argb[0] = ba; + argb[1] = rg; +#else + argb[0] = rg; + argb[1] = ba; +#endif +} + +#else // Table-free version (slower on x86) + +// These constants are 16b fixed-point version of ITU-R BT.601 constants +#define kYScale 76309 // 1.164 = 255 / 219 +#define kVToR 104597 // 1.596 = 255 / 112 * 0.701 +#define kUToG 25674 // 0.391 = 255 / 112 * 0.886 * 0.114 / 0.587 +#define kVToG 53278 // 0.813 = 255 / 112 * 0.701 * 0.299 / 0.587 +#define kUToB 132201 // 2.018 = 255 / 112 * 0.886 +#define kRCst (-kYScale * 16 - kVToR * 128 + YUV_HALF) +#define kGCst (-kYScale * 16 + kUToG * 128 + kVToG * 128 + YUV_HALF) +#define kBCst (-kYScale * 16 - kUToB * 128 + YUV_HALF) + +static WEBP_INLINE uint8_t VP8Clip8(int v) { + return ((v & ~YUV_MASK) == 0) ? (uint8_t)(v >> YUV_FIX) + : (v < 0) ? 0u : 255u; +} + +static WEBP_INLINE uint8_t VP8ClipN(int v, int N) { // clip to N bits + return ((v & ~YUV_MASK) == 0) ? (uint8_t)(v >> (YUV_FIX + (8 - N))) + : (v < 0) ? 0u : (255u >> (8 - N)); +} + +static WEBP_INLINE int VP8YUVToR(int y, int v) { + return kYScale * y + kVToR * v + kRCst; +} + +static WEBP_INLINE int VP8YUVToG(int y, int u, int v) { + return kYScale * y - kUToG * u - kVToG * v + kGCst; +} + +static WEBP_INLINE int VP8YUVToB(int y, int u) { + return kYScale * y + kUToB * u + kBCst; +} + +static WEBP_INLINE void VP8YuvToRgb(uint8_t y, uint8_t u, uint8_t v, + uint8_t* const rgb) { + rgb[0] = VP8Clip8(VP8YUVToR(y, v)); + rgb[1] = VP8Clip8(VP8YUVToG(y, u, v)); + rgb[2] = VP8Clip8(VP8YUVToB(y, u)); } static WEBP_INLINE void VP8YuvToBgr(uint8_t y, uint8_t u, uint8_t v, uint8_t* const bgr) { - const int r_off = VP8kVToR[v]; - const int g_off = (VP8kVToG[v] + VP8kUToG[u]) >> YUV_FIX; - const int b_off = VP8kUToB[u]; - bgr[0] = VP8kClip[y + b_off - YUV_RANGE_MIN]; - bgr[1] = VP8kClip[y + g_off - YUV_RANGE_MIN]; - bgr[2] = VP8kClip[y + r_off - YUV_RANGE_MIN]; + bgr[0] = VP8Clip8(VP8YUVToB(y, u)); + bgr[1] = VP8Clip8(VP8YUVToG(y, u, v)); + bgr[2] = VP8Clip8(VP8YUVToR(y, v)); +} + +static WEBP_INLINE void VP8YuvToRgb565(uint8_t y, uint8_t u, uint8_t v, + uint8_t* const rgb) { + const int r = VP8Clip8(VP8YUVToR(y, u)); + const int g = VP8ClipN(VP8YUVToG(y, u, v), 6); + const int b = VP8ClipN(VP8YUVToB(y, v), 5); + const uint8_t rg = (r & 0xf8) | (g >> 3); + const uint8_t gb = (g << 5) | b; +#ifdef WEBP_SWAP_16BIT_CSP + rgb[0] = gb; + rgb[1] = rg; +#else + rgb[0] = rg; + rgb[1] = gb; +#endif +} + +static WEBP_INLINE void VP8YuvToRgba4444(uint8_t y, uint8_t u, uint8_t v, + uint8_t* const argb) { + const int r = VP8Clip8(VP8YUVToR(y, u)); + const int g = VP8ClipN(VP8YUVToG(y, u, v), 4); + const int b = VP8Clip8(VP8YUVToB(y, v)); + const uint8_t rg = (r & 0xf0) | g; + const uint8_t ba = b | 0x0f; // overwrite the lower 4 bits +#ifdef WEBP_SWAP_16BIT_CSP + argb[0] = ba; + argb[1] = rg; +#else + argb[0] = rg; + argb[1] = ba; +#endif +} + +#endif // WEBP_YUV_USE_TABLE + +static WEBP_INLINE void VP8YuvToArgb(uint8_t y, uint8_t u, uint8_t v, + uint8_t* const argb) { + argb[0] = 0xff; + VP8YuvToRgb(y, u, v, argb + 1); } static WEBP_INLINE void VP8YuvToBgra(uint8_t y, uint8_t u, uint8_t v, @@ -90,28 +223,19 @@ static WEBP_INLINE void VP8YuvToRgba(uint8_t y, uint8_t u, uint8_t v, rgba[3] = 0xff; } -static WEBP_INLINE uint32_t VP8Clip4Bits(uint8_t c) { - const uint32_t v = (c + 8) >> 4; - return (v > 15) ? 15 : v; -} - // Must be called before everything, to initialize the tables. void VP8YUVInit(void); //------------------------------------------------------------------------------ // RGB -> YUV conversion -// The exact naming is Y'CbCr, following the ITU-R BT.601 standard. -// More information at: http://en.wikipedia.org/wiki/YCbCr -// Y = 0.2569 * R + 0.5044 * G + 0.0979 * B + 16 -// U = -0.1483 * R - 0.2911 * G + 0.4394 * B + 128 -// V = 0.4394 * R - 0.3679 * G - 0.0715 * B + 128 -// We use 16bit fixed point operations. static WEBP_INLINE int VP8ClipUV(int v) { - v = (v + (257 << (YUV_FIX + 2 - 1))) >> (YUV_FIX + 2); - return ((v & ~0xff) == 0) ? v : (v < 0) ? 0 : 255; + v = (v + (257 << (YUV_FIX + 2 - 1))) >> (YUV_FIX + 2); + return ((v & ~0xff) == 0) ? v : (v < 0) ? 0 : 255; } +#ifndef USE_YUVj + static WEBP_INLINE int VP8RGBToY(int r, int g, int b) { const int kRound = (1 << (YUV_FIX - 1)) + (16 << YUV_FIX); const int luma = 16839 * r + 33059 * g + 6420 * b; @@ -119,13 +243,38 @@ static WEBP_INLINE int VP8RGBToY(int r, int g, int b) { } static WEBP_INLINE int VP8RGBToU(int r, int g, int b) { - return VP8ClipUV(-9719 * r - 19081 * g + 28800 * b); + const int u = -9719 * r - 19081 * g + 28800 * b; + return VP8ClipUV(u); +} + +static WEBP_INLINE int VP8RGBToV(int r, int g, int b) { + const int v = +28800 * r - 24116 * g - 4684 * b; + return VP8ClipUV(v); +} + +#else + +// This JPEG-YUV colorspace, only for comparison! +// These are also 16-bit precision coefficients from Rec.601, but with full +// [0..255] output range. +static WEBP_INLINE int VP8RGBToY(int r, int g, int b) { + const int kRound = (1 << (YUV_FIX - 1)); + const int luma = 19595 * r + 38470 * g + 7471 * b; + return (luma + kRound) >> YUV_FIX; // no need to clip +} + +static WEBP_INLINE int VP8RGBToU(int r, int g, int b) { + const int u = -11058 * r - 21710 * g + 32768 * b; + return VP8ClipUV(u); } static WEBP_INLINE int VP8RGBToV(int r, int g, int b) { - return VP8ClipUV(+28800 * r - 24116 * g - 4684 * b); + const int v = 32768 * r - 27439 * g - 5329 * b; + return VP8ClipUV(v); } +#endif // USE_YUVj + #if defined(__cplusplus) || defined(c_plusplus) } // extern "C" #endif -- cgit v1.1