cc: Remove NaNs in MapEnclosingClippedRect and ViewportSelection

This CL removes NaNs in two places:
1) The mapped rect that MapEnclosingClippedRect passes to
   gfx::ToEnclosingRect. When this rect contains a NaN, it is now
   replaced with an empty rect.
2) The edge_top and edge_bottom values in the ViewportSelectionBounds
   computed in LayerTreeImpl::GetViewportSelection. When either of
   these values contains a NaN, an empty selection is produced
   instead.

These NaNs need to be removed since base::saturated_cast CHECKs when
asked to round a NaN to an int.

BUG=532379
CQ_INCLUDE_TRYBOTS=tryserver.blink:linux_blink_rel

Review URL: https://codereview.chromium.org/1357513003

Cr-Commit-Position: refs/heads/master@{#349487}

2 files changed, 115 insertions, 3 deletions

diff --git a/cc/base/math_util.cc b/cc/base/math_util.cc
index a91c474..6e5ac0b 100644
--- a/cc/base/math_util.cc
+++ b/cc/base/math_util.cc
@@ -127,7 +127,14 @@ gfx::Rect MathUtil::MapEnclosingClippedRect(const gfx::Transform& transform,
                          static_cast<int>(transform.matrix().getFloat(1, 3)));
     return src_rect + offset;
   }
-  return gfx::ToEnclosingRect(MapClippedRect(transform, gfx::RectF(src_rect)));
+  gfx::RectF mapped_rect = MapClippedRect(transform, gfx::RectF(src_rect));
+
+  // gfx::ToEnclosingRect crashes if called on a RectF with any NaN coordinate.
+  if (std::isnan(mapped_rect.x()) || std::isnan(mapped_rect.y()) ||
+      std::isnan(mapped_rect.right()) || std::isnan(mapped_rect.bottom()))
+    return gfx::Rect();
+
+  return gfx::ToEnclosingRect(mapped_rect);
 }
 
 gfx::RectF MathUtil::MapClippedRect(const gfx::Transform& transform,
@@ -167,8 +174,15 @@ gfx::Rect MathUtil::ProjectEnclosingClippedRect(const gfx::Transform& transform,
                          static_cast<int>(transform.matrix().getFloat(1, 3)));
     return src_rect + offset;
   }
-  return gfx::ToEnclosingRect(
-      ProjectClippedRect(transform, gfx::RectF(src_rect)));
+  gfx::RectF projected_rect =
+      ProjectClippedRect(transform, gfx::RectF(src_rect));
+
+  // gfx::ToEnclosingRect crashes if called on a RectF with any NaN coordinate.
+  if (std::isnan(projected_rect.x()) || std::isnan(projected_rect.y()) ||
+      std::isnan(projected_rect.right()) || std::isnan(projected_rect.bottom()))
+    return gfx::Rect();
+
+  return gfx::ToEnclosingRect(projected_rect);
 }
 
 gfx::RectF MathUtil::ProjectClippedRect(const gfx::Transform& transform,
diff --git a/cc/base/math_util_unittest.cc b/cc/base/math_util_unittest.cc
index ae0cf98..9d44fba 100644
--- a/cc/base/math_util_unittest.cc
+++ b/cc/base/math_util_unittest.cc
@@ -219,6 +219,104 @@ TEST(MathUtilTest, MapEnclosedRectWith2dAxisAlignedTransform) {
   EXPECT_EQ(gfx::Rect(2, 4, 6, 8), output);
 }
 
+TEST(MathUtilTest, MapEnclosingRectWithLargeTransforms) {
+  gfx::Rect input(1, 2, 100, 200);
+  gfx::Rect output;
+
+  gfx::Transform large_x_scale;
+  large_x_scale.Scale(SkDoubleToMScalar(1e37), 1.0);
+
+  gfx::Transform infinite_x_scale;
+  infinite_x_scale = large_x_scale * large_x_scale;
+
+  gfx::Transform large_y_scale;
+  large_y_scale.Scale(1.0, SkDoubleToMScalar(1e37));
+
+  gfx::Transform infinite_y_scale;
+  infinite_y_scale = large_y_scale * large_y_scale;
+
+  gfx::Transform rotation;
+  rotation.RotateAboutYAxis(170.0);
+
+  int max_int = std::numeric_limits<int>::max();
+
+  output = MathUtil::MapEnclosingClippedRect(large_x_scale, input);
+  EXPECT_EQ(gfx::Rect(max_int, 2, 0, 200), output);
+
+  output = MathUtil::MapEnclosingClippedRect(large_x_scale * rotation, input);
+  EXPECT_EQ(gfx::Rect(), output);
+
+  output = MathUtil::MapEnclosingClippedRect(infinite_x_scale, input);
+  EXPECT_EQ(gfx::Rect(max_int, 2, 0, 200), output);
+
+  output =
+      MathUtil::MapEnclosingClippedRect(infinite_x_scale * rotation, input);
+  EXPECT_EQ(gfx::Rect(), output);
+
+  output = MathUtil::MapEnclosingClippedRect(large_y_scale, input);
+  EXPECT_EQ(gfx::Rect(1, max_int, 100, 0), output);
+
+  output = MathUtil::MapEnclosingClippedRect(large_y_scale * rotation, input);
+  EXPECT_EQ(gfx::Rect(-100, max_int, 100, 0), output);
+
+  output = MathUtil::MapEnclosingClippedRect(infinite_y_scale, input);
+  EXPECT_EQ(gfx::Rect(1, max_int, 100, 0), output);
+
+  output =
+      MathUtil::MapEnclosingClippedRect(infinite_y_scale * rotation, input);
+  EXPECT_EQ(gfx::Rect(), output);
+}
+
+TEST(MathUtilTest, ProjectEnclosingRectWithLargeTransforms) {
+  gfx::Rect input(1, 2, 100, 200);
+  gfx::Rect output;
+
+  gfx::Transform large_x_scale;
+  large_x_scale.Scale(SkDoubleToMScalar(1e37), 1.0);
+
+  gfx::Transform infinite_x_scale;
+  infinite_x_scale = large_x_scale * large_x_scale;
+
+  gfx::Transform large_y_scale;
+  large_y_scale.Scale(1.0, SkDoubleToMScalar(1e37));
+
+  gfx::Transform infinite_y_scale;
+  infinite_y_scale = large_y_scale * large_y_scale;
+
+  gfx::Transform rotation;
+  rotation.RotateAboutYAxis(170.0);
+
+  int max_int = std::numeric_limits<int>::max();
+
+  output = MathUtil::ProjectEnclosingClippedRect(large_x_scale, input);
+  EXPECT_EQ(gfx::Rect(max_int, 2, 0, 200), output);
+
+  output =
+      MathUtil::ProjectEnclosingClippedRect(large_x_scale * rotation, input);
+  EXPECT_EQ(gfx::Rect(), output);
+
+  output = MathUtil::ProjectEnclosingClippedRect(infinite_x_scale, input);
+  EXPECT_EQ(gfx::Rect(max_int, 2, 0, 200), output);
+
+  output =
+      MathUtil::ProjectEnclosingClippedRect(infinite_x_scale * rotation, input);
+  EXPECT_EQ(gfx::Rect(), output);
+
+  output = MathUtil::ProjectEnclosingClippedRect(large_y_scale, input);
+  EXPECT_EQ(gfx::Rect(1, max_int, 100, 0), output);
+
+  output =
+      MathUtil::ProjectEnclosingClippedRect(large_y_scale * rotation, input);
+  EXPECT_EQ(gfx::Rect(-103, max_int, 102, 0), output);
+
+  output = MathUtil::ProjectEnclosingClippedRect(infinite_y_scale, input);
+  EXPECT_EQ(gfx::Rect(1, max_int, 100, 0), output);
+
+  output =
+      MathUtil::ProjectEnclosingClippedRect(infinite_y_scale * rotation, input);
+  EXPECT_EQ(gfx::Rect(), output);
+}
+
 TEST(MathUtilTest, RoundUp) {
   for (int multiplier = 1; multiplier <= 10; ++multiplier) {
     // Try attempts in descending order, so that we can