diff options
author | motek@chromium.org <motek@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2013-04-17 10:39:07 +0000 |
---|---|---|
committer | motek@chromium.org <motek@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2013-04-17 10:39:07 +0000 |
commit | bef5cc1a8083a1a495b4341dfeba5d4f01889f9e (patch) | |
tree | a956f26e0235dad20936a3f495dfb54842ec2f68 | |
parent | ca690b0ccb3684bec0a7a5ed2406346762b57ba7 (diff) | |
download | chromium_src-bef5cc1a8083a1a495b4341dfeba5d4f01889f9e.zip chromium_src-bef5cc1a8083a1a495b4341dfeba5d4f01889f9e.tar.gz chromium_src-bef5cc1a8083a1a495b4341dfeba5d4f01889f9e.tar.bz2 |
Complete (but inefficient) implementation of the image retargetting method.
This CL contains:
1. Convolution of arbitrary channel with arbitrary kernel (convolver*).
2. Gaussian gradient magnitude implementation.
3. Image profile (X and Y projections) computations.
4. Otsu-like thresholding of profiles.
5. Image decimation.
6. The main routine which binds it all together.
Note: 1 and 2 do work, but remain main sources of suckiness due to performance
problems. I actively work on this. Still, I'd love to get the current state in
to establish a baseline and for viewing pleasure of those who are interested.
BUG=155269
Review URL: https://codereview.chromium.org/13947013
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@194565 0039d316-1c4b-4281-b951-d872f2087c98
-rw-r--r-- | chrome/browser/thumbnails/content_analysis.cc | 452 | ||||
-rw-r--r-- | chrome/browser/thumbnails/content_analysis.h | 66 | ||||
-rw-r--r-- | chrome/browser/thumbnails/content_analysis_unittest.cc | 479 | ||||
-rw-r--r-- | chrome/chrome_browser.gypi | 2 | ||||
-rw-r--r-- | chrome/chrome_tests_unit.gypi | 1 | ||||
-rw-r--r-- | skia/ext/convolver.cc | 196 | ||||
-rw-r--r-- | skia/ext/convolver.h | 54 | ||||
-rw-r--r-- | skia/ext/convolver_unittest.cc | 149 |
8 files changed, 1396 insertions, 3 deletions
diff --git a/chrome/browser/thumbnails/content_analysis.cc b/chrome/browser/thumbnails/content_analysis.cc new file mode 100644 index 0000000..2a9d9ff --- /dev/null +++ b/chrome/browser/thumbnails/content_analysis.cc @@ -0,0 +1,452 @@ +// Copyright (c) 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/thumbnails/content_analysis.h" + +#include <algorithm> +#include <cmath> +#include <deque> +#include <functional> +#include <limits> +#include <numeric> +#include <vector> + +#include "base/logging.h" +#include "skia/ext/convolver.h" +#include "third_party/skia/include/core/SkBitmap.h" +#include "third_party/skia/include/core/SkSize.h" +#include "ui/gfx/color_analysis.h" + +namespace { + +template<class InputIterator, class OutputIterator, class Compare> +void SlidingWindowMinMax(InputIterator first, + InputIterator last, + OutputIterator output, + int window_size, + Compare cmp) { + typedef std::deque< + std::pair<typename std::iterator_traits<InputIterator>::value_type, int> > + deque_type; + deque_type slider; + int front_tail_length = window_size / 2; + int i = 0; + DCHECK_LT(front_tail_length, last - first); + // This min-max filter functions the way image filters do. The min/max we + // compute is placed in the center of the window. Thus, first we need to + // 'pre-load' the window with the slider with right-tail of the filter. + for (; first < last && i < front_tail_length; ++i, ++first) + slider.push_back(std::make_pair(*first, i)); + + for (; first < last; ++i, ++first, ++output) { + while (!slider.empty() && !cmp(slider.back().first, *first)) + slider.pop_back(); + slider.push_back(std::make_pair(*first, i)); + + while (slider.front().second <= i - window_size) + slider.pop_front(); + *output = slider.front().first; + } + + // Now at the tail-end we will simply need to use whatever value is left of + // the filter to compute the remaining front_tail_length taps in the output. + + // If input shorter than window, remainder length needs to be adjusted. + front_tail_length = std::min(front_tail_length, i); + for (; front_tail_length >= 0; --front_tail_length, ++i) { + while (slider.front().second <= i - window_size) + slider.pop_front(); + *output = slider.front().first; + } +} + +} // namespace + +namespace thumbnailing_utils { + +void ApplyGaussianGradientMagnitudeFilter(SkBitmap* input_bitmap, + float kernel_sigma) { + // The purpose of this function is to highlight salient + // (attention-attracting?) features of the image for use in image + // retargetting. + SkAutoLockPixels source_lock(*input_bitmap); + DCHECK(input_bitmap); + DCHECK(input_bitmap->getPixels()); + DCHECK_EQ(SkBitmap::kA8_Config, input_bitmap->config()); + + const int tail_length = static_cast<int>(4.0f * kernel_sigma + 0.5f); + const int kernel_size = tail_length * 2 + 1; + const float sigmasq = kernel_sigma * kernel_sigma; + std::vector<float> smoother_weights(kernel_size, 0.0); + float kernel_sum = 1.0f; + + smoother_weights[tail_length] = 1.0f; + + for (int ii = 1; ii <= tail_length; ++ii) { + float v = std::exp(-0.5f * ii * ii / sigmasq); + smoother_weights[tail_length + ii] = v; + smoother_weights[tail_length - ii] = v; + kernel_sum += 2.0f * v; + } + + for (int i = 0; i < kernel_size; ++i) + smoother_weights[i] /= kernel_sum; + + std::vector<float> gradient_weights(kernel_size, 0.0); + gradient_weights[tail_length] = 0.0; + for (int ii = 1; ii <= tail_length; ++ii) { + float v = sigmasq * smoother_weights[tail_length + ii] / ii; + gradient_weights[tail_length + ii] = v; + gradient_weights[tail_length - ii] = -v; + } + + skia::ConvolutionFilter1D smoothing_filter; + skia::ConvolutionFilter1D gradient_filter; + smoothing_filter.AddFilter(0, &smoother_weights[0], smoother_weights.size()); + gradient_filter.AddFilter(0, &gradient_weights[0], gradient_weights.size()); + + // To perform computations we will need one intermediate buffer. It can + // very well be just another bitmap. + const SkISize image_size = SkISize::Make(input_bitmap->width(), + input_bitmap->height()); + SkBitmap intermediate; + intermediate.setConfig( + input_bitmap->config(), image_size.width(), image_size.height()); + intermediate.allocPixels(); + + skia::SingleChannelConvolveX1D( + input_bitmap->getAddr8(0, 0), + static_cast<int>(input_bitmap->rowBytes()), + 0, input_bitmap->bytesPerPixel(), + smoothing_filter, + image_size, + intermediate.getAddr8(0, 0), + static_cast<int>(intermediate.rowBytes()), + 0, intermediate.bytesPerPixel(), false); + skia::SingleChannelConvolveY1D( + intermediate.getAddr8(0, 0), + static_cast<int>(intermediate.rowBytes()), + 0, intermediate.bytesPerPixel(), + smoothing_filter, + image_size, + input_bitmap->getAddr8(0, 0), + static_cast<int>(input_bitmap->rowBytes()), + 0, input_bitmap->bytesPerPixel(), false); + + // Now the gradient operator (we will need two buffers). + SkBitmap intermediate2; + intermediate2.setConfig( + input_bitmap->config(), image_size.width(), image_size.height()); + intermediate2.allocPixels(); + + skia::SingleChannelConvolveX1D( + input_bitmap->getAddr8(0, 0), + static_cast<int>(input_bitmap->rowBytes()), + 0, input_bitmap->bytesPerPixel(), + gradient_filter, + image_size, + intermediate.getAddr8(0, 0), + static_cast<int>(intermediate.rowBytes()), + 0, intermediate.bytesPerPixel(), true); + skia::SingleChannelConvolveY1D( + input_bitmap->getAddr8(0, 0), + static_cast<int>(input_bitmap->rowBytes()), + 0, input_bitmap->bytesPerPixel(), + gradient_filter, + image_size, + intermediate2.getAddr8(0, 0), + static_cast<int>(intermediate2.rowBytes()), + 0, intermediate2.bytesPerPixel(), true); + + unsigned grad_max = 0; + for (int r = 0; r < image_size.height(); ++r) { + const uint8* grad_x_row = intermediate.getAddr8(0, r); + const uint8* grad_y_row = intermediate2.getAddr8(0, r); + for (int c = 0; c < image_size.width(); ++c) { + unsigned grad_x = grad_x_row[c]; + unsigned grad_y = grad_y_row[c]; + grad_max = std::max(grad_max, grad_x * grad_x + grad_y * grad_y); + } + } + + int bit_shift = 0; + if (grad_max > 255) + bit_shift = static_cast<int>( + std::log10(static_cast<float>(grad_max)) / std::log10(2.0f)) - 7; + for (int r = 0; r < image_size.height(); ++r) { + const uint8* grad_x_row =intermediate.getAddr8(0, r); + const uint8* grad_y_row = intermediate2.getAddr8(0, r); + uint8* target_row = input_bitmap->getAddr8(0, r); + for (int c = 0; c < image_size.width(); ++c) { + unsigned grad_x = grad_x_row[c]; + unsigned grad_y = grad_y_row[c]; + target_row[c] = (grad_x * grad_x + grad_y * grad_y) >> bit_shift; + } + } +} + +void ExtractImageProfileInformation(const SkBitmap& input_bitmap, + const gfx::Rect& area, + const gfx::Size& target_size, + bool apply_log, + std::vector<float>* rows, + std::vector<float>* columns) { + SkAutoLockPixels source_lock(input_bitmap); + DCHECK(rows); + DCHECK(columns); + DCHECK(input_bitmap.getPixels()); + DCHECK_EQ(SkBitmap::kA8_Config, input_bitmap.config()); + DCHECK_GE(area.x(), 0); + DCHECK_GE(area.y(), 0); + DCHECK_LE(area.right(), input_bitmap.width()); + DCHECK_LE(area.bottom(), input_bitmap.height()); + + // Make sure rows and columns are allocated and initialized to 0. + rows->clear(); + columns->clear(); + rows->resize(area.height(), 0); + columns->resize(area.width(), 0); + + for (int r = 0; r < area.height(); ++r) { + // Points to the first byte of the row in the rectangle. + const uint8* image_row = input_bitmap.getAddr8(area.x(), r + area.y()); + unsigned row_sum = 0; + for (int c = 0; c < area.width(); ++c, ++image_row) { + row_sum += *image_row; + (*columns)[c] += *image_row; + } + (*rows)[r] = row_sum; + } + + if (apply_log) { + // Generally for processing we will need to take logarithm of this data. + // The option not to apply it is left principally as a test seam. + std::vector<float>::iterator it; + for (it = columns->begin(); it < columns->end(); ++it) + *it = std::log(1.0f + *it); + + for (it = rows->begin(); it < rows->end(); ++it) + *it = std::log(1.0f + *it); + } + + if (!target_size.IsEmpty()) { + // If the target size is given, profiles should be further processed through + // morphological closing. The idea is to close valleys smaller than what + // can be seen after scaling down to avoid deforming noticable features + // when profiles are used. + // Morphological closing is defined as dilation followed by errosion. In + // normal-speak: sliding-window maximum followed by minimum. + int column_window_size = 1 + 2 * + static_cast<int>(0.5f * area.width() / target_size.width() + 0.5f); + int row_window_size = 1 + 2 * + static_cast<int>(0.5f * area.height() / target_size.height() + 0.5f); + + // Dilate and erode each profile with the given window size. + if (column_window_size >= 3) { + SlidingWindowMinMax(columns->begin(), + columns->end(), + columns->begin(), + column_window_size, + std::greater<float>()); + SlidingWindowMinMax(columns->begin(), + columns->end(), + columns->begin(), + column_window_size, + std::less<float>()); + } + + if (row_window_size >= 3) { + SlidingWindowMinMax(rows->begin(), + rows->end(), + rows->begin(), + row_window_size, + std::greater<float>()); + SlidingWindowMinMax(rows->begin(), + rows->end(), + rows->begin(), + row_window_size, + std::less<float>()); + } + } +} + +float AutoSegmentPeaks(const std::vector<float>& input) { + // This is a thresholding operation based on Otsu's method. + std::vector<int> histogram(256, 0); + std::vector<float>::const_iterator it; + + float value_min = std::numeric_limits<float>::max(); + float value_max = std::numeric_limits<float>::min(); + + for (it = input.begin(); it < input.end(); ++it) { + value_min = std::min(value_min, *it); + value_max = std::max(value_max, *it); + } + + if (value_max - value_min <= std::numeric_limits<float>::epsilon() * 100) { + // Scaling won't work and there is nothing really to segment anyway. + return value_min; + } + + float value_span = value_max - value_min; + for (it = input.begin(); it < input.end(); ++it) { + float scaled_value = (*it - value_min) / value_span * 255; + histogram[static_cast<int>(scaled_value)] += 1; + } + + // Otsu's method seeks to maximize variance between two classes of pixels + // correspondng to valleys and peaks of the profile. + double w1 = histogram[0]; // Total weight of the first class. + double t1 = 0.5 * w1; + double w2 = 0; + double t2 = 0; + for (size_t i = 1; i < histogram.size(); ++i) { + w2 += histogram[i]; + t2 += (0.5 + i) * histogram[i]; + } + + size_t max_index = 0; + double m1 = t1 / w1; + double m2 = t2 / w2; + double max_variance_score = w1 * w2 * (m1 - m2) * (m1 - m2); + // Iterate through all possible ways of splitting the histogram. + for (size_t i = 1; i < histogram.size() - 1; i++) { + double bin_volume = (0.5 + i) * histogram[i]; + w1 += histogram[i]; + w2 -= histogram[i]; + t2 -= bin_volume; + t1 += bin_volume; + m1 = t1 / w1; + m2 = t2 / w2; + double variance_score = w1 * w2 * (m1 - m2) * (m1 - m2); + if (variance_score >= max_variance_score) { + max_variance_score = variance_score; + max_index = i; + } + } + + // max_index refers to the bin *after* which we need to split. The sought + // threshold is the centre of this bin, scaled back to the original range. + return value_span * (max_index + 0.5f) / 255.0f + value_min; +} + +SkBitmap ComputeDecimatedImage(const SkBitmap& bitmap, + const std::vector<bool>& rows, + const std::vector<bool>& columns) { + SkAutoLockPixels source_lock(bitmap); + DCHECK(bitmap.getPixels()); + DCHECK_GT(bitmap.bytesPerPixel(), 0); + DCHECK_EQ(bitmap.width(), static_cast<int>(columns.size())); + DCHECK_EQ(bitmap.height(), static_cast<int>(rows.size())); + + unsigned target_row_count = std::count(rows.begin(), rows.end(), true); + unsigned target_column_count = std::count( + columns.begin(), columns.end(), true); + + if (target_row_count == 0 || target_column_count == 0) + return SkBitmap(); // Not quite an error, so no DCHECK. Just return empty. + + if (target_row_count == rows.size() && target_column_count == columns.size()) + return SkBitmap(); // Equivalent of the situation above (empty target). + + // Allocate the target image. + SkBitmap target; + target.setConfig(bitmap.config(), target_column_count, target_row_count); + target.allocPixels(); + + int target_row = 0; + for (int r = 0; r < bitmap.height(); ++r) { + if (!rows[r]) + continue; // We can just skip this one. + uint8* src_row = + static_cast<uint8*>(bitmap.getPixels()) + r * bitmap.rowBytes(); + uint8* insertion_target = static_cast<uint8*>(target.getPixels()) + + target_row * target.rowBytes(); + int left_copy_pixel = -1; + for (int c = 0; c < bitmap.width(); ++c) { + if (left_copy_pixel < 0 && columns[c]) { + left_copy_pixel = c; // Next time we will start copying from here. + } else if (left_copy_pixel >= 0 && !columns[c]) { + // This closes a fragment we want to copy. We do it now. + size_t bytes_to_copy = (c - left_copy_pixel) * bitmap.bytesPerPixel(); + memcpy(insertion_target, + src_row + left_copy_pixel * bitmap.bytesPerPixel(), + bytes_to_copy); + left_copy_pixel = -1; + insertion_target += bytes_to_copy; + } + } + // We can still have the tail end to process here. + if (left_copy_pixel >= 0) { + size_t bytes_to_copy = + (bitmap.width() - left_copy_pixel) * bitmap.bytesPerPixel(); + memcpy(insertion_target, + src_row + left_copy_pixel * bitmap.bytesPerPixel(), + bytes_to_copy); + } + target_row++; + } + + return target; +} + +SkBitmap CreateRetargettedThumbnailImage( + const SkBitmap& source_bitmap, + const gfx::Size& target_size, + float kernel_sigma) { + // First thing we need for this method is to color-reduce the source_bitmap. + SkBitmap reduced_color; + reduced_color.setConfig( + SkBitmap::kA8_Config, source_bitmap.width(), source_bitmap.height()); + reduced_color.allocPixels(); + + if (!color_utils::ComputePrincipalComponentImage(source_bitmap, + &reduced_color)) { + // CCIR601 luminance conversion vector. + gfx::Vector3dF transform(0.299f, 0.587f, 0.114f); + if (!color_utils::ApplyColorReduction( + source_bitmap, transform, true, &reduced_color)) { + DLOG(WARNING) << "Failed to compute luminance image from a screenshot. " + << "Cannot compute retargetted thumbnail."; + return SkBitmap(); + } + DLOG(WARNING) << "Could not compute principal color image for a thumbnail. " + << "Using luminance instead."; + } + + // Turn 'color-reduced' image into the 'energy' image. + ApplyGaussianGradientMagnitudeFilter(&reduced_color, kernel_sigma); + + // Extract vertical and horizontal projection of image features. + std::vector<float> row_profile; + std::vector<float> column_profile; + ExtractImageProfileInformation(reduced_color, + gfx::Rect(reduced_color.width(), + reduced_color.height()), + target_size, + true, + &row_profile, + &column_profile); + float threshold_rows = AutoSegmentPeaks(row_profile); + float threshold_columns = AutoSegmentPeaks(column_profile); + + // Apply thresholding. + std::vector<bool> included_rows(row_profile.size(), false); + std::transform(row_profile.begin(), + row_profile.end(), + included_rows.begin(), + std::bind2nd(std::greater<float>(), threshold_rows)); + + std::vector<bool> included_columns(column_profile.size(), false); + std::transform(column_profile.begin(), + column_profile.end(), + included_columns.begin(), + std::bind2nd(std::greater<float>(), threshold_columns)); + + // Use the original image and computed inclusion vectors to create a resized + // image. + return ComputeDecimatedImage(source_bitmap, included_rows, included_columns); +} + +} // thumbnailing_utils diff --git a/chrome/browser/thumbnails/content_analysis.h b/chrome/browser/thumbnails/content_analysis.h new file mode 100644 index 0000000..e65ff02 --- /dev/null +++ b/chrome/browser/thumbnails/content_analysis.h @@ -0,0 +1,66 @@ +// Copyright (c) 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_BROWSER_THUMBNAILS_CONTENT_ANALYSIS_H_ +#define CHROME_BROWSER_THUMBNAILS_CONTENT_ANALYSIS_H_ + +#include <vector> + +#include "base/basictypes.h" +#include "third_party/skia/include/core/SkColor.h" +#include "ui/base/ui_export.h" +#include "ui/gfx/rect.h" +#include "ui/gfx/size.h" + +class SkBitmap; + +namespace thumbnailing_utils { + +// Compute in-place gaussian gradient magnitude of |input_bitmap| with sigma +// |kernel_sigma|. |input_bitmap| is requried to be of SkBitmap::kA8_Config +// type. The routine computes first-order gaussian derivative on a +// gaussian-smoothed image. Beware, this is fairly slow since kernel size is +// 4 * kernel_sigma + 1. +void ApplyGaussianGradientMagnitudeFilter(SkBitmap* input_bitmap, + float kernel_sigma); + +// Accumulates vertical and horizontal sum of pixel values from a subsection of +// |input_bitmap| defined by |image_area|. The image is required to be of +// SkBitmap::kA8_Config type. +// If non-empty |target_size| is given, the routine will use it to process the +// profiles with closing operator sized to eliminate gaps which would be smaller +// than 1 pixel after rescaling to |target_size|. +// If |apply_log| is true, logarithm of counts are used for morhology (and +// returned). +void ExtractImageProfileInformation(const SkBitmap& input_bitmap, + const gfx::Rect& image_area, + const gfx::Size& target_size, + bool apply_log, + std::vector<float>* rows, + std::vector<float>* columns); + +// Compute a threshold value separating background (low) from signal (high) +// areas in the |input| profile. +float AutoSegmentPeaks(const std::vector<float>& input); + +// Shrinks the source |bitmap| by removing rows and columns where |rows| and +// |columns| are false, respectively. The function returns a new bitmap if the +// shrinking can be performed and an empty instance otherwise. +SkBitmap ComputeDecimatedImage(const SkBitmap& bitmap, + const std::vector<bool>& rows, + const std::vector<bool>& columns); + +// Creates a new bitmap which contains only 'interesting' areas of +// |source_bitmap|. The |target_size| is used to estimate some computation +// parameters, but the resulting bitmap will not necessarily be of that size. +// |kernel_sigma| defines the degree of image smoothing in gradient computation. +// For a natural-sized (not shrunk) screenshot at 96 DPI and regular font size +// 5.0 was determined to be a good value. +SkBitmap CreateRetargettedThumbnailImage(const SkBitmap& source_bitmap, + const gfx::Size& target_size, + float kernel_sigma); + +} // namespace thumbnailing_utils + +#endif // CHROME_BROWSER_THUMBNAILS_CONTENT_ANALYSIS_H_ diff --git a/chrome/browser/thumbnails/content_analysis_unittest.cc b/chrome/browser/thumbnails/content_analysis_unittest.cc new file mode 100644 index 0000000..77b9bcb --- /dev/null +++ b/chrome/browser/thumbnails/content_analysis_unittest.cc @@ -0,0 +1,479 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/thumbnails/content_analysis.h" + +#include <algorithm> +#include <cmath> +#include <cstdlib> +#include <limits> +#include <numeric> +#include <vector> + +#include "base/memory/scoped_ptr.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/skia/include/core/SkBitmap.h" +#include "third_party/skia/include/core/SkColor.h" +#include "ui/gfx/canvas.h" +#include "ui/gfx/color_analysis.h" +#include "ui/gfx/color_utils.h" +#include "ui/gfx/image/image.h" +#include "ui/gfx/rect.h" +#include "ui/gfx/size.h" + +namespace { + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +unsigned long ImagePixelSum(const SkBitmap& bitmap, const gfx::Rect& rect) { + // Get the sum of pixel values in the rectangle. Applicable only to + // monochrome bitmaps. + DCHECK_EQ(SkBitmap::kA8_Config, bitmap.config()); + unsigned long total = 0; + for (int r = rect.y(); r < rect.bottom(); ++r) { + const uint8* row_data = static_cast<const uint8*>( + bitmap.getPixels()) + r * bitmap.rowBytes(); + for (int c = rect.x(); c < rect.right(); ++c) + total += row_data[c]; + } + + return total; +} + +bool CompareImageFragments(const SkBitmap& bitmap_left, + const SkBitmap& bitmap_right, + const gfx::Size& comparison_area, + const gfx::Point& origin_left, + const gfx::Point& origin_right) { + SkAutoLockPixels left_lock(bitmap_left); + SkAutoLockPixels right_lock(bitmap_right); + for (int r = 0; r < comparison_area.height(); ++r) { + for (int c = 0; c < comparison_area.width(); ++c) { + SkColor color_left = bitmap_left.getColor(origin_left.x() + c, + origin_left.y() + r); + SkColor color_right = bitmap_right.getColor(origin_right.x() + c, + origin_right.y() + r); + if (color_left != color_right) + return false; + } + } + + return true; +} + +} // namespace + +namespace thumbnailing_utils { + +class ThumbnailContentAnalysisTest : public testing::Test { +}; + +TEST_F(ThumbnailContentAnalysisTest, ApplyGradientMagnitudeOnImpulse) { + gfx::Canvas canvas(gfx::Size(800, 600), ui::SCALE_FACTOR_100P, true); + + // The image consists of vertical non-overlapping stripes 100 pixels wide. + canvas.FillRect(gfx::Rect(0, 0, 800, 600), SkColorSetARGB(0, 10, 10, 10)); + canvas.FillRect(gfx::Rect(400, 300, 1, 1), SkColorSetRGB(255, 255, 255)); + + SkBitmap source = + skia::GetTopDevice(*canvas.sk_canvas())->accessBitmap(false); + + SkBitmap reduced_color; + reduced_color.setConfig( + SkBitmap::kA8_Config, source.width(), source.height()); + reduced_color.allocPixels(); + + gfx::Vector3dF transform(0.299f, 0.587f, 0.114f); + EXPECT_TRUE(color_utils::ApplyColorReduction( + source, transform, true, &reduced_color)); + + float sigma = 2.5f; + ApplyGaussianGradientMagnitudeFilter(&reduced_color, sigma); + + // Expect everything to be within 8 * sigma. + int tail_length = static_cast<int>(8.0f * sigma + 0.5f); + gfx::Rect echo_rect(399 - tail_length, 299 - tail_length, + 2 * tail_length + 1, 2 * tail_length + 1); + unsigned long data_sum = ImagePixelSum(reduced_color, echo_rect); + unsigned long all_sum = ImagePixelSum(reduced_color, gfx::Rect(800, 600)); + EXPECT_GT(data_sum, 0U); + EXPECT_EQ(data_sum, all_sum); + + sigma = 5.0f; + ApplyGaussianGradientMagnitudeFilter(&reduced_color, sigma); + + // Expect everything to be within 8 * sigma. + tail_length = static_cast<int>(8.0f * sigma + 0.5f); + echo_rect = gfx::Rect(399 - tail_length, 299 - tail_length, + 2 * tail_length + 1, 2 * tail_length + 1); + data_sum = ImagePixelSum(reduced_color, echo_rect); + all_sum = ImagePixelSum(reduced_color, gfx::Rect(800, 600)); + EXPECT_GT(data_sum, 0U); + EXPECT_EQ(data_sum, all_sum); +} + +TEST_F(ThumbnailContentAnalysisTest, ApplyGradientMagnitudeOnFrame) { + gfx::Canvas canvas(gfx::Size(800, 600), ui::SCALE_FACTOR_100P, true); + + // The image consists of a single white block in the centre. + gfx::Rect draw_rect(300, 200, 200, 200); + canvas.FillRect(gfx::Rect(0, 0, 800, 600), SkColorSetARGB(0, 0, 0, 0)); + canvas.DrawRect(draw_rect, SkColorSetRGB(255, 255, 255)); + + SkBitmap source = + skia::GetTopDevice(*canvas.sk_canvas())->accessBitmap(false); + + SkBitmap reduced_color; + reduced_color.setConfig( + SkBitmap::kA8_Config, source.width(), source.height()); + reduced_color.allocPixels(); + + gfx::Vector3dF transform(0.299f, 0.587f, 0.114f); + EXPECT_TRUE(color_utils::ApplyColorReduction( + source, transform, true, &reduced_color)); + + float sigma = 2.5f; + ApplyGaussianGradientMagnitudeFilter(&reduced_color, sigma); + + int tail_length = static_cast<int>(8.0f * sigma + 0.5f); + gfx::Rect outer_rect(draw_rect.x() - tail_length, + draw_rect.y() - tail_length, + draw_rect.width() + 2 * tail_length, + draw_rect.height() + 2 * tail_length); + gfx::Rect inner_rect(draw_rect.x() + tail_length, + draw_rect.y() + tail_length, + draw_rect.width() - 2 * tail_length, + draw_rect.height() - 2 * tail_length); + unsigned long data_sum = ImagePixelSum(reduced_color, outer_rect); + unsigned long all_sum = ImagePixelSum(reduced_color, gfx::Rect(800, 600)); + EXPECT_GT(data_sum, 0U); + EXPECT_EQ(data_sum, all_sum); + EXPECT_EQ(ImagePixelSum(reduced_color, inner_rect), 0U); +} + +TEST_F(ThumbnailContentAnalysisTest, ExtractImageProfileInformation) { + gfx::Canvas canvas(gfx::Size(800, 600), ui::SCALE_FACTOR_100P, true); + + // The image consists of a white frame drawn in the centre. + gfx::Rect draw_rect(100, 100, 200, 100); + gfx::Rect image_rect(0, 0, 800, 600); + canvas.FillRect(image_rect, SkColorSetARGB(0, 0, 0, 0)); + canvas.DrawRect(draw_rect, SkColorSetRGB(255, 255, 255)); + + SkBitmap source = + skia::GetTopDevice(*canvas.sk_canvas())->accessBitmap(false); + SkBitmap reduced_color; + reduced_color.setConfig( + SkBitmap::kA8_Config, source.width(), source.height()); + reduced_color.allocPixels(); + + gfx::Vector3dF transform(1, 0, 0); + EXPECT_TRUE(color_utils::ApplyColorReduction( + source, transform, true, &reduced_color)); + std::vector<float> column_profile; + std::vector<float> row_profile; + ExtractImageProfileInformation(reduced_color, + image_rect, + gfx::Size(), + false, + &row_profile, + &column_profile); + EXPECT_EQ(0, std::accumulate(column_profile.begin(), + column_profile.begin() + draw_rect.x() - 1, + 0)); + EXPECT_EQ(column_profile[draw_rect.x()], 255U * (draw_rect.height() + 1)); + EXPECT_EQ(2 * 255 * (draw_rect.width() - 2), + std::accumulate(column_profile.begin() + draw_rect.x() + 1, + column_profile.begin() + draw_rect.right() - 1, + 0)); + + EXPECT_EQ(0, std::accumulate(row_profile.begin(), + row_profile.begin() + draw_rect.y() - 1, + 0)); + EXPECT_EQ(row_profile[draw_rect.y()], 255U * (draw_rect.width() + 1)); + EXPECT_EQ(2 * 255 * (draw_rect.height() - 2), + std::accumulate(row_profile.begin() + draw_rect.y() + 1, + row_profile.begin() + draw_rect.bottom() - 1, + 0)); + + gfx::Rect test_rect(150, 80, 400, 100); + ExtractImageProfileInformation(reduced_color, + test_rect, + gfx::Size(), + false, + &row_profile, + &column_profile); + + // Some overlap with the drawn rectagle. If you work it out on a piece of + // paper, sums should be as follows. + EXPECT_EQ(255 * (test_rect.bottom() - draw_rect.y()) + + 255 * (draw_rect.right() - test_rect.x()), + std::accumulate(row_profile.begin(), row_profile.end(), 0)); + EXPECT_EQ(255 * (test_rect.bottom() - draw_rect.y()) + + 255 * (draw_rect.right() - test_rect.x()), + std::accumulate(column_profile.begin(), column_profile.end(), 0)); +} + +TEST_F(ThumbnailContentAnalysisTest, + ExtractImageProfileInformationWithClosing) { + gfx::Canvas canvas(gfx::Size(800, 600), ui::SCALE_FACTOR_100P, true); + + // The image consists of a two white frames drawn side by side, with a + // single-pixel vertical gap in between. + gfx::Rect image_rect(0, 0, 800, 600); + canvas.FillRect(image_rect, SkColorSetARGB(0, 0, 0, 0)); + canvas.DrawRect(gfx::Rect(300, 250, 99, 100), SkColorSetRGB(255, 255, 255)); + canvas.DrawRect(gfx::Rect(401, 250, 99, 100), SkColorSetRGB(255, 255, 255)); + + SkBitmap source = + skia::GetTopDevice(*canvas.sk_canvas())->accessBitmap(false); + SkBitmap reduced_color; + reduced_color.setConfig( + SkBitmap::kA8_Config, source.width(), source.height()); + reduced_color.allocPixels(); + + gfx::Vector3dF transform(1, 0, 0); + EXPECT_TRUE(color_utils::ApplyColorReduction( + source, transform, true, &reduced_color)); + std::vector<float> column_profile; + std::vector<float> row_profile; + + ExtractImageProfileInformation(reduced_color, + image_rect, + gfx::Size(), + true, + &row_profile, + &column_profile); + // Column profiles should have two spikes in the middle, with a single + // 0-valued value between them. + EXPECT_GT(column_profile[398], 0.0f); + EXPECT_GT(column_profile[399], column_profile[398]); + EXPECT_GT(column_profile[402], 0.0f); + EXPECT_GT(column_profile[401], column_profile[402]); + EXPECT_EQ(column_profile[401], column_profile[399]); + EXPECT_EQ(column_profile[402], column_profile[398]); + EXPECT_EQ(column_profile[400], 0.0f); + EXPECT_EQ(column_profile[299], 0.0f); + EXPECT_EQ(column_profile[502], 0.0f); + + // Now the same with closing applied. The space in the middle will be closed. + ExtractImageProfileInformation(reduced_color, + image_rect, + gfx::Size(200, 100), + true, + &row_profile, + &column_profile); + EXPECT_GT(column_profile[398], 0); + EXPECT_GT(column_profile[400], 0); + EXPECT_GT(column_profile[402], 0); + EXPECT_EQ(column_profile[299], 0); + EXPECT_EQ(column_profile[502], 0); + EXPECT_EQ(column_profile[399], column_profile[401]); + EXPECT_EQ(column_profile[398], column_profile[402]); +} + +TEST_F(ThumbnailContentAnalysisTest, AutoSegmentPeaks) { + std::vector<float> profile_info; + + EXPECT_EQ(AutoSegmentPeaks(profile_info), std::numeric_limits<float>::max()); + profile_info.resize(1000, 1.0f); + EXPECT_EQ(AutoSegmentPeaks(profile_info), 1.0f); + std::srand(42); + std::generate(profile_info.begin(), profile_info.end(), std::rand); + float threshold = AutoSegmentPeaks(profile_info); + EXPECT_GT(threshold, 0); // Not much to expect. + + // There should be roughly 50% above and below the threshold. + // Random is not really random thanks to srand, so we can sort-of compare. + int above_count = std::count_if( + profile_info.begin(), + profile_info.end(), + std::bind2nd(std::greater<float>(), threshold)); + EXPECT_GT(above_count, 450); // Not much to expect. + EXPECT_LT(above_count, 550); + + for (unsigned i = 0; i < profile_info.size(); ++i) { + float y = std::sin(M_PI * i / 250.0f); + profile_info[i] = y > 0 ? y : 0; + } + threshold = AutoSegmentPeaks(profile_info); + + above_count = std::count_if( + profile_info.begin(), + profile_info.end(), + std::bind2nd(std::greater<float>(), threshold)); + EXPECT_LT(above_count, 500); // Negative y expected to fall below threshold. + + // Expect two peaks around between 0 and 250 and 500 and 750. + std::vector<bool> thresholded_values(profile_info.size(), false); + std::transform(profile_info.begin(), + profile_info.end(), + thresholded_values.begin(), + std::bind2nd(std::greater<float>(), threshold)); + EXPECT_TRUE(thresholded_values[125]); + EXPECT_TRUE(thresholded_values[625]); + int transitions = 0; + for (unsigned i = 1; i < thresholded_values.size(); ++i) { + if (thresholded_values[i] != thresholded_values[i-1]) + transitions++; + } + EXPECT_EQ(transitions, 4); // We have two contiguous peaks. Good going! +} + +TEST_F(ThumbnailContentAnalysisTest, ComputeDecimatedImage) { + gfx::Size image_size(1600, 1200); + gfx::Canvas canvas(image_size, ui::SCALE_FACTOR_100P, true); + + // Make some content we will later want to keep. + canvas.FillRect(gfx::Rect(100, 200, 100, 100), SkColorSetARGB(0, 125, 0, 0)); + canvas.FillRect(gfx::Rect(300, 200, 100, 100), SkColorSetARGB(0, 0, 200, 0)); + canvas.FillRect(gfx::Rect(500, 200, 100, 100), SkColorSetARGB(0, 0, 0, 225)); + canvas.FillRect(gfx::Rect(100, 400, 600, 100), + SkColorSetARGB(0, 125, 200, 225)); + + std::vector<bool> rows(image_size.height(), false); + std::fill_n(rows.begin() + 200, 100, true); + std::fill_n(rows.begin() + 400, 100, true); + + std::vector<bool> columns(image_size.width(), false); + std::fill_n(columns.begin() + 100, 100, true); + std::fill_n(columns.begin() + 300, 100, true); + std::fill_n(columns.begin() + 500, 100, true); + + SkBitmap source = + skia::GetTopDevice(*canvas.sk_canvas())->accessBitmap(false); + SkBitmap result = ComputeDecimatedImage(source, rows, columns); + EXPECT_FALSE(result.empty()); + EXPECT_EQ(300, result.width()); + EXPECT_EQ(200, result.height()); + + // The call should have removed all empty spaces. + ASSERT_TRUE(CompareImageFragments(source, + result, + gfx::Size(100, 100), + gfx::Point(100, 200), + gfx::Point(0, 0))); + ASSERT_TRUE(CompareImageFragments(source, + result, + gfx::Size(100, 100), + gfx::Point(300, 200), + gfx::Point(100, 0))); + ASSERT_TRUE(CompareImageFragments(source, + result, + gfx::Size(100, 100), + gfx::Point(500, 200), + gfx::Point(200, 0))); + ASSERT_TRUE(CompareImageFragments(source, + result, + gfx::Size(100, 100), + gfx::Point(100, 400), + gfx::Point(0, 0))); +} + +TEST_F(ThumbnailContentAnalysisTest, CreateRetargettedThumbnailImage) { + gfx::Size image_size(1200, 1300); + gfx::Canvas canvas(image_size, ui::SCALE_FACTOR_100P, true); + + // The following will create a 'fake image' consisting of color blocks placed + // on a neutral background. The entire layout is supposed to mimic a + // screenshot of a web page. + // The tested function is supposed to locate the interesing areas in the + // middle. + const int margin_horizontal = 60; + const int margin_vertical = 20; + canvas.FillRect(gfx::Rect(image_size), SkColorSetRGB(200, 210, 210)); + const gfx::Rect header_rect(margin_horizontal, + margin_vertical, + image_size.width() - 2 * margin_horizontal, + 100); + const gfx::Rect footer_rect(margin_horizontal, + image_size.height() - margin_vertical - 100, + image_size.width() - 2 * margin_horizontal, + 100); + const gfx::Rect body_rect(margin_horizontal, + header_rect.bottom() + margin_vertical, + image_size.width() - 2 * margin_horizontal, + footer_rect.y() - header_rect.bottom() - + 2 * margin_vertical); + canvas.FillRect(header_rect, SkColorSetRGB(200, 40, 10)); + canvas.FillRect(footer_rect, SkColorSetRGB(10, 40, 180)); + canvas.FillRect(body_rect, SkColorSetRGB(150, 180, 40)); + + // 'Fine print' at the bottom. + const int fine_print = 8; + const SkColor print_color = SkColorSetRGB(45, 30, 30); + for (int y = footer_rect.y() + fine_print; + y < footer_rect.bottom() - fine_print; + y += 2 * fine_print) { + for (int x = footer_rect.x() + fine_print; + x < footer_rect.right() - fine_print; + x += 2 * fine_print) { + canvas.DrawRect(gfx::Rect(x, y, fine_print, fine_print), print_color); + } + } + + // Blocky content at the top. + const int block_size = header_rect.height() - margin_vertical; + for (int x = header_rect.x() + margin_horizontal; + x < header_rect.right() - block_size; + x += block_size + margin_horizontal) { + const int half_block = block_size / 2 - 5; + const SkColor block_color = SkColorSetRGB(255, 255, 255); + const int y = header_rect.y() + margin_vertical / 2; + int second_col = x + half_block + 10; + int second_row = y + half_block + 10; + canvas.FillRect(gfx::Rect(x, y, half_block, block_size), block_color); + canvas.FillRect(gfx::Rect(second_col, y, half_block, half_block), + block_color); + canvas.FillRect(gfx::Rect(second_col, second_row, half_block, half_block), + block_color); + } + + // Now the main body. Mostly text with some 'pictures'. + for (int y = body_rect.y() + fine_print; + y < body_rect.bottom() - fine_print; + y += 2 * fine_print) { + for (int x = body_rect.x() + fine_print; + x < body_rect.right() - fine_print; + x += 2 * fine_print) { + canvas.DrawRect(gfx::Rect(x, y, fine_print, fine_print), print_color); + } + } + + for (int line = 0; line < 3; ++line) { + int alignment = line % 2; + const int y = body_rect.y() + + body_rect.height() / 3 * line + margin_vertical; + const int x = body_rect.x() + + alignment * body_rect.width() / 2 + margin_vertical; + gfx::Rect pict_rect(x, y, + body_rect.width() / 2 - 2 * margin_vertical, + body_rect.height() / 3 - 2 * margin_vertical); + canvas.FillRect(pict_rect, SkColorSetRGB(255, 255, 255)); + canvas.DrawRect(pict_rect, SkColorSetRGB(0, 0, 0)); + } + + SkBitmap source = + skia::GetTopDevice(*canvas.sk_canvas())->accessBitmap(false); + + SkBitmap result = CreateRetargettedThumbnailImage( + source, gfx::Size(424, 264), 2.5); + EXPECT_FALSE(result.empty()); + + // Given the nature of computation We can't really assert much here about the + // image itself. We know it should have been computed, should be smaller than + // the original and it must not be zero. + EXPECT_LT(result.width(), image_size.width()); + EXPECT_LT(result.height(), image_size.height()); + + int histogram[256]; + color_utils::BuildLumaHistogram(result, histogram); + int non_zero_color_count = std::count_if( + histogram, histogram + 256, std::bind2nd(std::greater<int>(), 0)); + EXPECT_GT(non_zero_color_count, 4); +} + +} // namespace thumbnailing_utils diff --git a/chrome/chrome_browser.gypi b/chrome/chrome_browser.gypi index cda5d6c..ae165f2 100644 --- a/chrome/chrome_browser.gypi +++ b/chrome/chrome_browser.gypi @@ -2042,6 +2042,8 @@ 'browser/themes/theme_syncable_service.h', 'browser/three_d_api_observer.cc', 'browser/three_d_api_observer.h', + 'browser/thumbnails/content_analysis.cc', + 'browser/thumbnails/content_analysis.h', 'browser/thumbnails/simple_thumbnail_crop.cc', 'browser/thumbnails/simple_thumbnail_crop.h', 'browser/thumbnails/render_widget_snapshot_taker.cc', diff --git a/chrome/chrome_tests_unit.gypi b/chrome/chrome_tests_unit.gypi index e8ed314..b1e162c 100644 --- a/chrome/chrome_tests_unit.gypi +++ b/chrome/chrome_tests_unit.gypi @@ -1191,6 +1191,7 @@ 'browser/themes/theme_properties_unittest.cc', 'browser/themes/theme_service_unittest.cc', 'browser/themes/theme_syncable_service_unittest.cc', + 'browser/thumbnails/content_analysis_unittest.cc', 'browser/thumbnails/render_widget_snapshot_taker_unittest.cc', 'browser/thumbnails/simple_thumbnail_crop_unittest.cc', 'browser/thumbnails/thumbnail_service_unittest.cc', diff --git a/skia/ext/convolver.cc b/skia/ext/convolver.cc index cbfa931..2057b22 100644 --- a/skia/ext/convolver.cc +++ b/skia/ext/convolver.cc @@ -4,8 +4,10 @@ #include <algorithm> +#include "base/logging.h" #include "skia/ext/convolver.h" #include "skia/ext/convolver_SSE2.h" +#include "third_party/skia/include/core/SkSize.h" #include "third_party/skia/include/core/SkTypes.h" namespace skia { @@ -22,6 +24,17 @@ inline unsigned char ClampTo8(int a) { return 255; } +// Takes the value produced by accumulating element-wise product of image with +// a kernel and brings it back into range. +// All of the filter scaling factors are in fixed point with kShiftBits bits of +// fractional part. +inline unsigned char BringBackTo8(int a, bool take_absolute) { + a >>= ConvolutionFilter1D::kShiftBits; + if (take_absolute) + a = std::abs(a); + return ClampTo8(a); +} + // Stores a list of rows in a circular buffer. The usage is you write into it // by calling AdvanceRow. It will keep track of which row in the buffer it // should use next, and the total number of rows added. @@ -271,6 +284,7 @@ void ConvolutionFilter1D::AddFilter(int filter_offset, // cases it is beneficial to only store the central factors. // For a scaling to 1/4th in each dimension using a Lanczos-2 filter on // a 1080p image this optimization gives a ~10% speed improvement. + int filter_size = filter_length; int first_non_zero = 0; while (first_non_zero < filter_length && filter_values[first_non_zero] == 0) first_non_zero++; @@ -298,12 +312,27 @@ void ConvolutionFilter1D::AddFilter(int filter_offset, instance.data_location = (static_cast<int>(filter_values_.size()) - filter_length); instance.offset = filter_offset; - instance.length = filter_length; + instance.trimmed_length = filter_length; + instance.length = filter_size; filters_.push_back(instance); max_filter_ = std::max(max_filter_, filter_length); } +const ConvolutionFilter1D::Fixed* ConvolutionFilter1D::GetSingleFilter( + int* specified_filter_length, + int* filter_offset, + int* filter_length) const { + const FilterInstance& filter = filters_[0]; + *filter_offset = filter.offset; + *filter_length = filter.trimmed_length; + *specified_filter_length = filter.length; + if (filter.trimmed_length == 0) + return NULL; + + return &filter_values_[filter.data_location]; +} + typedef void (*ConvolveVertically_pointer)( const ConvolutionFilter1D::Fixed* filter_values, int filter_length, @@ -478,4 +507,169 @@ void BGRAConvolve2D(const unsigned char* source_data, } } +void SingleChannelConvolveX1D(const unsigned char* source_data, + int source_byte_row_stride, + int input_channel_index, + int input_channel_count, + const ConvolutionFilter1D& filter, + const SkISize& image_size, + unsigned char* output, + int output_byte_row_stride, + int output_channel_index, + int output_channel_count, + bool absolute_values) { + int filter_offset, filter_length, filter_size; + // Very much unlike BGRAConvolve2D, here we expect to have the same filter + // for all pixels. + const ConvolutionFilter1D::Fixed* filter_values = + filter.GetSingleFilter(&filter_size, &filter_offset, &filter_length); + + if (filter_values == NULL || image_size.width() < filter_size) { + NOTREACHED(); + return; + } + + int centrepoint = filter_length / 2; + if (filter_size - filter_offset != 2 * filter_offset) { + // This means the original filter was not symmetrical AND + // got clipped from one side more than from the other. + centrepoint = filter_size / 2 - filter_offset; + } + + const unsigned char* source_data_row = source_data; + unsigned char* output_row = output; + + for (int r = 0; r < image_size.height(); ++r) { + unsigned char* target_byte = output_row + output_channel_index; + // Process the lead part, padding image to the left with the first pixel. + int c = 0; + for (; c < centrepoint; ++c, target_byte += output_channel_count) { + int accval = 0; + int i = 0; + int pixel_byte_index = input_channel_index; + for (; i < centrepoint - c; ++i) // Padding part. + accval += filter_values[i] * source_data_row[pixel_byte_index]; + + for (; i < filter_length; ++i, pixel_byte_index += input_channel_count) + accval += filter_values[i] * source_data_row[pixel_byte_index]; + + *target_byte = BringBackTo8(accval, absolute_values); + } + + // Now for the main event. + for (; c < image_size.width() - centrepoint; + ++c, target_byte += output_channel_count) { + int accval = 0; + int pixel_byte_index = (c - centrepoint) * input_channel_count + + input_channel_index; + + for (int i = 0; i < filter_length; + ++i, pixel_byte_index += input_channel_count) { + accval += filter_values[i] * source_data_row[pixel_byte_index]; + } + + *target_byte = BringBackTo8(accval, absolute_values); + } + + for (; c < image_size.width(); ++c, target_byte += output_channel_count) { + int accval = 0; + int overlap_taps = image_size.width() - c + centrepoint; + int pixel_byte_index = (c - centrepoint) * input_channel_count + + input_channel_index; + int i = 0; + for (; i < overlap_taps - 1; ++i, pixel_byte_index += input_channel_count) + accval += filter_values[i] * source_data_row[pixel_byte_index]; + + for (; i < filter_length; ++i) + accval += filter_values[i] * source_data_row[pixel_byte_index]; + + *target_byte = BringBackTo8(accval, absolute_values); + } + + source_data_row += source_byte_row_stride; + output_row += output_byte_row_stride; + } +} + +void SingleChannelConvolveY1D(const unsigned char* source_data, + int source_byte_row_stride, + int input_channel_index, + int input_channel_count, + const ConvolutionFilter1D& filter, + const SkISize& image_size, + unsigned char* output, + int output_byte_row_stride, + int output_channel_index, + int output_channel_count, + bool absolute_values) { + int filter_offset, filter_length, filter_size; + // Very much unlike BGRAConvolve2D, here we expect to have the same filter + // for all pixels. + const ConvolutionFilter1D::Fixed* filter_values = + filter.GetSingleFilter(&filter_size, &filter_offset, &filter_length); + + if (filter_values == NULL || image_size.height() < filter_size) { + NOTREACHED(); + return; + } + + int centrepoint = filter_length / 2; + if (filter_size - filter_offset != 2 * filter_offset) { + // This means the original filter was not symmetrical AND + // got clipped from one side more than from the other. + centrepoint = filter_size / 2 - filter_offset; + } + + for (int c = 0; c < image_size.width(); ++c) { + unsigned char* target_byte = output + c * output_channel_count + + output_channel_index; + int r = 0; + + for (; r < centrepoint; ++r, target_byte += output_byte_row_stride) { + int accval = 0; + int i = 0; + int pixel_byte_index = c * input_channel_count + input_channel_index; + + for (; i < centrepoint - r; ++i) // Padding part. + accval += filter_values[i] * source_data[pixel_byte_index]; + + for (; i < filter_length; ++i, pixel_byte_index += source_byte_row_stride) + accval += filter_values[i] * source_data[pixel_byte_index]; + + *target_byte = BringBackTo8(accval, absolute_values); + } + + for (; r < image_size.height() - centrepoint; + ++r, target_byte += output_byte_row_stride) { + int accval = 0; + int pixel_byte_index = (r - centrepoint) * source_byte_row_stride + + c * input_channel_count + input_channel_index; + for (int i = 0; i < filter_length; + ++i, pixel_byte_index += source_byte_row_stride) { + accval += filter_values[i] * source_data[pixel_byte_index]; + } + + *target_byte = BringBackTo8(accval, absolute_values); + } + + for (; r < image_size.height(); + ++r, target_byte += output_byte_row_stride) { + int accval = 0; + int overlap_taps = image_size.height() - r + centrepoint; + int pixel_byte_index = (r - centrepoint) * source_byte_row_stride + + c * input_channel_count + input_channel_index; + int i = 0; + for (; i < overlap_taps - 1; + ++i, pixel_byte_index += source_byte_row_stride) { + accval += filter_values[i] * source_data[pixel_byte_index]; + } + + for (; i < filter_length; ++i) + accval += filter_values[i] * source_data[pixel_byte_index]; + + *target_byte = BringBackTo8(accval, absolute_values); + } + } +} + } // namespace skia diff --git a/skia/ext/convolver.h b/skia/ext/convolver.h index 3065338..6da703c 100644 --- a/skia/ext/convolver.h +++ b/skia/ext/convolver.h @@ -10,6 +10,7 @@ #include "base/basictypes.h" #include "base/cpu.h" +#include "third_party/skia/include/core/SkSize.h" #include "third_party/skia/include/core/SkTypes.h" // We can build SSE2 optimized versions for all x86 CPUs @@ -99,13 +100,24 @@ class ConvolutionFilter1D { int* filter_length) const { const FilterInstance& filter = filters_[value_offset]; *filter_offset = filter.offset; - *filter_length = filter.length; - if (filter.length == 0) { + *filter_length = filter.trimmed_length; + if (filter.trimmed_length == 0) { return NULL; } return &filter_values_[filter.data_location]; } + // Retrieves the filter for the offset 0, presumed to be the one and only. + // The offset and length of the filter values are put into the corresponding + // out arguments (see AddFilter). Note that |filter_legth| and + // |specified_filter_length| may be different if leading/trailing zeros of the + // original floating point form were clipped. + // There will be |filter_length| values in the return array. + // Returns NULL if the filter is 0-length (for instance when all floating + // point values passed to AddFilter were clipped to 0). + const Fixed* GetSingleFilter(int* specified_filter_length, + int* filter_offset, + int* filter_length) const; inline void PaddingForSIMD() { // Padding |padding_count| of more dummy coefficients after the coefficients @@ -128,6 +140,11 @@ class ConvolutionFilter1D { int offset; // Number of values in this filter instance. + int trimmed_length; + + // Filter length as specified. Note that this may be different from + // 'trimmed_length' if leading/trailing zeros of the original floating + // point form were clipped differently on each tail. int length; }; @@ -169,6 +186,39 @@ SK_API void BGRAConvolve2D(const unsigned char* source_data, int output_byte_row_stride, unsigned char* output, bool use_simd_if_possible); + +// Does a 1D convolution of the given source image along the X dimension on +// a single channel of the bitmap. +// +// The function uses the same convolution kernel for each pixel. That kernel +// must be added to |filter| at offset 0. This is a most straightforward +// implementation of convolution, intended chiefly for development purposes. +SK_API void SingleChannelConvolveX1D(const unsigned char* source_data, + int source_byte_row_stride, + int input_channel_index, + int input_channel_count, + const ConvolutionFilter1D& filter, + const SkISize& image_size, + unsigned char* output, + int output_byte_row_stride, + int output_channel_index, + int output_channel_count, + bool absolute_values); + +// Does a 1D convolution of the given source image along the Y dimension on +// a single channel of the bitmap. +SK_API void SingleChannelConvolveY1D(const unsigned char* source_data, + int source_byte_row_stride, + int input_channel_index, + int input_channel_count, + const ConvolutionFilter1D& filter, + const SkISize& image_size, + unsigned char* output, + int output_byte_row_stride, + int output_channel_index, + int output_channel_count, + bool absolute_values); + } // namespace skia #endif // SKIA_EXT_CONVOLVER_H_ diff --git a/skia/ext/convolver_unittest.cc b/skia/ext/convolver_unittest.cc index 377ed8e..b877655 100644 --- a/skia/ext/convolver_unittest.cc +++ b/skia/ext/convolver_unittest.cc @@ -324,4 +324,153 @@ TEST(Convolver, SIMDVerification) { } } +TEST(Convolver, SeparableSingleConvolution) { + static const int kImgWidth = 1024; + static const int kImgHeight = 1024; + static const int kChannelCount = 3; + static const int kStrideSlack = 22; + ConvolutionFilter1D filter; + const float box[5] = { 0.2f, 0.2f, 0.2f, 0.2f, 0.2f }; + filter.AddFilter(0, box, 5); + + // Allocate a source image and set to 0. + const int src_row_stride = kImgWidth * kChannelCount + kStrideSlack; + int src_byte_count = src_row_stride * kImgHeight; + std::vector<unsigned char> input; + const int signal_x = kImgWidth / 2; + const int signal_y = kImgHeight / 2; + input.resize(src_byte_count, 0); + // The image has a single impulse pixel in channel 1, smack in the middle. + const int non_zero_pixel_index = + signal_y * src_row_stride + signal_x * kChannelCount + 1; + input[non_zero_pixel_index] = 255; + + // Destination will be a single channel image with stide matching width. + const int dest_row_stride = kImgWidth; + const int dest_byte_count = dest_row_stride * kImgHeight; + std::vector<unsigned char> output; + output.resize(dest_byte_count); + + // Apply convolution in X. + SingleChannelConvolveX1D(&input[0], src_row_stride, 1, kChannelCount, + filter, SkISize::Make(kImgWidth, kImgHeight), + &output[0], dest_row_stride, 0, 1, false); + for (int x = signal_x - 2; x <= signal_x + 2; ++x) + EXPECT_GT(output[signal_y * dest_row_stride + x], 0); + + EXPECT_EQ(output[signal_y * dest_row_stride + signal_x - 3], 0); + EXPECT_EQ(output[signal_y * dest_row_stride + signal_x + 3], 0); + + // Apply convolution in Y. + SingleChannelConvolveY1D(&input[0], src_row_stride, 1, kChannelCount, + filter, SkISize::Make(kImgWidth, kImgHeight), + &output[0], dest_row_stride, 0, 1, false); + for (int y = signal_y - 2; y <= signal_y + 2; ++y) + EXPECT_GT(output[y * dest_row_stride + signal_x], 0); + + EXPECT_EQ(output[(signal_y - 3) * dest_row_stride + signal_x], 0); + EXPECT_EQ(output[(signal_y + 3) * dest_row_stride + signal_x], 0); + + EXPECT_EQ(output[signal_y * dest_row_stride + signal_x - 1], 0); + EXPECT_EQ(output[signal_y * dest_row_stride + signal_x + 1], 0); + + // The main point of calling this is to invoke the routine on input without + // padding. + std::vector<unsigned char> output2; + output2.resize(dest_byte_count); + SingleChannelConvolveX1D(&output[0], dest_row_stride, 0, 1, + filter, SkISize::Make(kImgWidth, kImgHeight), + &output2[0], dest_row_stride, 0, 1, false); + // This should be a result of 2D convolution. + for (int x = signal_x - 2; x <= signal_x + 2; ++x) { + for (int y = signal_y - 2; y <= signal_y + 2; ++y) + EXPECT_GT(output2[y * dest_row_stride + x], 0); + } + EXPECT_EQ(output2[0], 0); + EXPECT_EQ(output2[dest_row_stride - 1], 0); + EXPECT_EQ(output2[dest_byte_count - 1], 0); +} + +TEST(Convolver, SeparableSingleConvolutionEdges) { + // The purpose of this test is to check if the implementation treats correctly + // edges of the image. + static const int kImgWidth = 600; + static const int kImgHeight = 800; + static const int kChannelCount = 3; + static const int kStrideSlack = 22; + static const int kChannel = 1; + ConvolutionFilter1D filter; + const float box[5] = { 0.2f, 0.2f, 0.2f, 0.2f, 0.2f }; + filter.AddFilter(0, box, 5); + + // Allocate a source image and set to 0. + int src_row_stride = kImgWidth * kChannelCount + kStrideSlack; + int src_byte_count = src_row_stride * kImgHeight; + std::vector<unsigned char> input(src_byte_count); + + // Draw a frame around the image. + for (int i = 0; i < src_byte_count; ++i) { + int row = i / src_row_stride; + int col = i % src_row_stride / kChannelCount; + int channel = i % src_row_stride % kChannelCount; + if (channel != kChannel || col > kImgWidth) { + input[i] = 255; + } else if (row == 0 || col == 0 || + col == kImgWidth - 1 || row == kImgHeight - 1) { + input[i] = 100; + } else if (row == 1 || col == 1 || + col == kImgWidth - 2 || row == kImgHeight - 2) { + input[i] = 200; + } else { + input[i] = 0; + } + } + + // Destination will be a single channel image with stide matching width. + int dest_row_stride = kImgWidth; + int dest_byte_count = dest_row_stride * kImgHeight; + std::vector<unsigned char> output; + output.resize(dest_byte_count); + + // Apply convolution in X. + SingleChannelConvolveX1D(&input[0], src_row_stride, 1, kChannelCount, + filter, SkISize::Make(kImgWidth, kImgHeight), + &output[0], dest_row_stride, 0, 1, false); + + // Sadly, comparison is not as simple as retaining all values. + int invalid_values = 0; + const unsigned char first_value = output[0]; + EXPECT_TRUE(std::abs(100 - first_value) <= 1); + for (int i = 0; i < dest_row_stride; ++i) { + if (output[i] != first_value) + ++invalid_values; + } + EXPECT_EQ(0, invalid_values); + + int test_row = 22; + EXPECT_NEAR(output[test_row * dest_row_stride], 100, 1); + EXPECT_NEAR(output[test_row * dest_row_stride + 1], 80, 1); + EXPECT_NEAR(output[test_row * dest_row_stride + 2], 60, 1); + EXPECT_NEAR(output[test_row * dest_row_stride + 3], 40, 1); + EXPECT_NEAR(output[(test_row + 1) * dest_row_stride - 1], 100, 1); + EXPECT_NEAR(output[(test_row + 1) * dest_row_stride - 2], 80, 1); + EXPECT_NEAR(output[(test_row + 1) * dest_row_stride - 3], 60, 1); + EXPECT_NEAR(output[(test_row + 1) * dest_row_stride - 4], 40, 1); + + SingleChannelConvolveY1D(&input[0], src_row_stride, 1, kChannelCount, + filter, SkISize::Make(kImgWidth, kImgHeight), + &output[0], dest_row_stride, 0, 1, false); + + int test_column = 42; + EXPECT_NEAR(output[test_column], 100, 1); + EXPECT_NEAR(output[test_column + dest_row_stride], 80, 1); + EXPECT_NEAR(output[test_column + dest_row_stride * 2], 60, 1); + EXPECT_NEAR(output[test_column + dest_row_stride * 3], 40, 1); + + EXPECT_NEAR(output[test_column + dest_row_stride * (kImgHeight - 1)], 100, 1); + EXPECT_NEAR(output[test_column + dest_row_stride * (kImgHeight - 2)], 80, 1); + EXPECT_NEAR(output[test_column + dest_row_stride * (kImgHeight - 3)], 60, 1); + EXPECT_NEAR(output[test_column + dest_row_stride * (kImgHeight - 4)], 40, 1); +} + } // namespace skia |