Swipe selection on Virtual Keyboard

This is a swipe selection feature on virtual keyboard on ChromeOS. This feature will improve user experience on text editing on touch screen and make the modification of text and position-setting of cursor much easier. It requires to implement an API and a swipe gesture recognizer on virtual keyboard.
How to trigger this?
TEST = 
1. Enable the virtual keyboard feature on ChromeOS.
2. To move the cursor horizontally, swipe horizontally from anywhere on the virtual keyboard.
3. To move the cursor vertically, swipe vertically from anywhere on the virtual keyboard.
4. To move the cursor on diagonal direction, swipe diagonal from anywhere on the virtual keyboard in the four directions, which means, upleft, upright, downleft, downright.
5. To move the cursor while selecting the text on the edit field, swipe from shift key as the start point to any directions you want.



BUG=263426

Review URL: https://chromiumcodereview.appspot.com/20818002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@216752 0039d316-1c4b-4281-b951-d872f2087c98

5 files changed, 225 insertions, 3 deletions

diff --git a/ui/keyboard/keyboard_util.cc b/ui/keyboard/keyboard_util.cc
index 076eacf..1579068 100644
--- a/ui/keyboard/keyboard_util.cc
+++ b/ui/keyboard/keyboard_util.cc
@@ -64,6 +64,45 @@ bool InsertText(const base::string16& text, aura::RootWindow* root_window) {
   return true;
 }
 
+// TODO(varunjain): It would be cleaner to have something in the
+// ui::TextInputClient interface, say MoveCaretInDirection(). The code in
+// here would get the ui::InputMethod from the root_window, and the
+// ui::TextInputClient from that (see above in InsertText()).
+bool MoveCursor(int swipe_direction,
+                int modifier_flags,
+                aura::RootWindow* root_window) {
+  if (!root_window)
+    return false;
+  ui::KeyboardCode codex = ui::VKEY_UNKNOWN;
+  ui::KeyboardCode codey = ui::VKEY_UNKNOWN;
+  if (swipe_direction & kCursorMoveRight)
+    codex = ui::VKEY_RIGHT;
+  else if (swipe_direction & kCursorMoveLeft)
+    codex = ui::VKEY_LEFT;
+
+  if (swipe_direction & kCursorMoveUp)
+    codey = ui::VKEY_UP;
+  else if (swipe_direction & kCursorMoveDown)
+    codey = ui::VKEY_DOWN;
+
+  // First deal with the x movement.
+  if (codex != ui::VKEY_UNKNOWN) {
+    ui::KeyEvent press_event(ui::ET_KEY_PRESSED, codex, modifier_flags, 0);
+    root_window->AsRootWindowHostDelegate()->OnHostKeyEvent(&press_event);
+    ui::KeyEvent release_event(ui::ET_KEY_RELEASED, codex, modifier_flags, 0);
+    root_window->AsRootWindowHostDelegate()->OnHostKeyEvent(&release_event);
+  }
+
+  // Then deal with the y movement.
+  if (codey != ui::VKEY_UNKNOWN) {
+    ui::KeyEvent press_event(ui::ET_KEY_PRESSED, codey, modifier_flags, 0);
+    root_window->AsRootWindowHostDelegate()->OnHostKeyEvent(&press_event);
+    ui::KeyEvent release_event(ui::ET_KEY_RELEASED, codey, modifier_flags, 0);
+    root_window->AsRootWindowHostDelegate()->OnHostKeyEvent(&release_event);
+  }
+  return true;
+}
+
 const GritResourceMap* GetKeyboardExtensionResources(size_t* size) {
   // This looks a lot like the contents of a resource map; however it is
   // necessary to have a custom path for the extension path, so the resource
diff --git a/ui/keyboard/keyboard_util.h b/ui/keyboard/keyboard_util.h
index 0fc60ff..7d46433 100644
--- a/ui/keyboard/keyboard_util.h
+++ b/ui/keyboard/keyboard_util.h
@@ -17,6 +17,14 @@ struct GritResourceMap;
 
 namespace keyboard {
 
+// Enumeration of swipe directions.
+enum CursorMoveDirection {
+  kCursorMoveRight = 0x01,
+  kCursorMoveLeft = 0x02,
+  kCursorMoveUp = 0x04,
+  kCursorMoveDown = 0x08
+};
+
 // Returns true if the virtual keyboard is enabled.
 KEYBOARD_EXPORT bool IsKeyboardEnabled();
 
@@ -27,6 +35,12 @@ KEYBOARD_EXPORT bool IsKeyboardEnabled();
 KEYBOARD_EXPORT bool InsertText(const base::string16& text,
                                 aura::RootWindow* root_window);
 
+// Move cursor when swipe on the virtualkeyboard. Returns true if cursor was
+// successfully moved according to |swipe_direction|.
+KEYBOARD_EXPORT bool MoveCursor(int swipe_direction,
+                                int modifier_flags,
+                                aura::RootWindow* root_window);
+
 // Get the list of keyboard resources.  |size| is populated with the number of
 // resources in the returned array.
 KEYBOARD_EXPORT const GritResourceMap* GetKeyboardExtensionResources(
diff --git a/ui/keyboard/resources/api_adapter.js b/ui/keyboard/resources/api_adapter.js
index cd64f57..eb95e35 100644
--- a/ui/keyboard/resources/api_adapter.js
+++ b/ui/keyboard/resources/api_adapter.js
@@ -11,3 +11,8 @@ function logIfError() {
 function insertText(text) {
   chrome.experimental.input.virtualKeyboard.insertText(text, logIfError);
 }
+
+function MoveCursor(swipe_direction, swipe_flags) {
+  chrome.experimental.input.virtualKeyboard.moveCursor(swipe_direction,
+                                                       swipe_flags);
+}
diff --git a/ui/keyboard/resources/elements/kb-key.html b/ui/keyboard/resources/elements/kb-key.html
index b8bbd4d..fc366bb 100644
--- a/ui/keyboard/resources/elements/kb-key.html
+++ b/ui/keyboard/resources/elements/kb-key.html
@@ -40,6 +40,7 @@
           detail.nextKeyset = this.keysetRules.dbl[NEXT_KEYSET - OFFSET];
         }
         this.fire('enable-dbl', detail);
+        this.fire('enable-sel');
       }
     });
   </script>
diff --git a/ui/keyboard/resources/elements/kb-keyboard.html b/ui/keyboard/resources/elements/kb-keyboard.html
index d21edf0..f4efb31 100644
--- a/ui/keyboard/resources/elements/kb-keyboard.html
+++ b/ui/keyboard/resources/elements/kb-keyboard.html
@@ -6,6 +6,7 @@
 
 <polymer-element name="kb-keyboard" on-key-over="keyOver" on-key-up="keyUp"
     on-key-down="keyDown" on-key-longpress="keyLongpress" on-pointerup="up"
+    on-pointerdown="down" on-enable-sel="enableSel"
     on-enable-dbl="enableDbl" attributes="keyset layout rows">
   <template>
     <style>
@@ -60,6 +61,7 @@
      * A structure to track the currently repeating key on the keyboard.
      */
     var repeatKey = {
+
       /**
         * The timer for the delay before repeating behaviour begins.
         * @type {number|undefined}
@@ -90,14 +92,97 @@
       }
     };
 
+    /**
+     * The minimum movement interval needed to trigger cursor move on
+     * horizontal and vertical way.
+     * @const
+     * @type {number}
+     */
+    var MIN_SWIPE_DIST = 30;
+
+    /**
+     * The flags constants when shift is on. It is according to the EventFlags
+     * in event_constants.h in chromium c++ code.
+     * @const
+     * @type {number}
+     * TODO(zyaozhujun): Might add more flags here according to the defination
+     * in EventFlags.
+     */
+    var SHIFT = 2;
+
+    /**
+     * The boolean to decide if it is swipe in process or finished.
+     * @const
+     * @type {boolean}
+     */
+    var swipeInProgress = false;
+
+    /**
+     * The enumeration of swipe directions.
+     * @const
+     * @type {Enum}
+     */
+    var SWIPE_DIRECTION = {
+      RIGHT: 0x1,
+      LEFT: 0x2,
+      UP: 0x4,
+      DOWN: 0x8
+    };
+
+    /**
+     * A structure to track the current swipe status.
+     */
+    var swipeStatus = {
+
+      /**
+       * The count of horizontal and vertical movement.
+       * @type {number}
+       */
+       offset_x : 0,
+       offset_y : 0,
+
+      /**
+       * Last touch coordinate.
+       * @type {number}
+       */
+      pre_x : 0,
+      pre_y : 0,
+
+      /**
+       * The flag of current modifier key.
+       * @type {number}
+       */
+      swipeFlags : 0,
+
+      /**
+       * Current swipe direction.
+       * @type {number}
+       */
+      swipeDirection : 0,
+
+      /**
+       * Reset all the values when swipe finished.
+       */
+      resetAll: function() {
+        this.offset_x = 0;
+        this.offset_y = 0;
+        this.pre_x = 0;
+        this.pre_y = 0;
+	this.swipeFlags = 0;
+	this.swipeDirection = 0;
+      }
+    };
+
     Polymer('kb-keyboard', {
       lastPressedKey: null,
       voiceInput_: null,
       dblDetail_: null,
       dblTimer_: null,
+      swipeHandler: null,
 
       ready: function() {
         this.voiceInput_ = new VoiceInput(this);
+        this.swipeHandler = this.onSwipeUpdate.bind(this);
       },
 
       /**
@@ -121,6 +206,48 @@
       },
 
       /**
+       * This function is bound to swipeHandler. And swipeHandler handle
+       * the pointermove event after pointerdown event happened.
+       * @para {PointerEvent} event.
+       */
+      onSwipeUpdate: function(event) {
+	swipeStatus.offset_x += event.screenX - swipeStatus.pre_x;
+	swipeStatus.offset_y += event.screenY - swipeStatus.pre_y;
+        if (Math.abs(swipeStatus.offset_x) > MIN_SWIPE_DIST ||
+            Math.abs(swipeStatus.offset_y) > MIN_SWIPE_DIST) {
+          swipeInProgress = true;
+          this.lastPressedKey.classList.remove('active');
+        }
+        if (swipeStatus.offset_x > MIN_SWIPE_DIST) {
+	  swipeStatus.swipeDirection |= SWIPE_DIRECTION.RIGHT;
+          swipeStatus.offset_x = 0;
+        }
+        else if (swipeStatus.offset_x < -MIN_SWIPE_DIST) {
+	  swipeStatus.swipeDirection |= SWIPE_DIRECTION.LEFT;
+	  swipeStatus.offset_x = 0;
+	}
+        // Swipe vertically only when the swipe reaches the gradient of 45
+        // degree. This can also be larger.
+	if (Math.abs(event.screenY - swipeStatus.pre_y) >
+            Math.abs(event.screenX - swipeStatus.pre_x)) {
+          if (swipeStatus.offset_y > MIN_SWIPE_DIST) {
+            swipeStatus.swipeDirection |= SWIPE_DIRECTION.DOWN;
+	    swipeStatus.offset_y = 0;
+	  }
+          else if (swipeStatus.offset_y < -MIN_SWIPE_DIST) {
+	    swipeStatus.swipeDirection |= SWIPE_DIRECTION.UP;
+            swipeStatus.offset_y = 0;
+	  }
+        }
+       if (swipeStatus.swipeDirection) {
+	  MoveCursor(swipeStatus.swipeDirection, swipeStatus.swipeFlags);
+          swipeStatus.swipeDirection = 0;
+        }
+	swipeStatus.pre_x = event.screenX;
+	swipeStatus.pre_y = event.screenY;
+      },
+
+      /**
        * Handles key-down event that is sent by kb-key-base.
        * @param {CustomEvent} event The key-down event dispatched by
        *     kb-key-base.
@@ -173,12 +300,34 @@
       },
 
       /**
+       * Enable the selection while swipe.
+       * @param {CustomEvent} event The enable-dbl event dispatched by
+       *    kb-shift-key.
+       */
+      enableSel: function(event) {
+        swipeStatus.swipeFlags = SHIFT;
+      },
+
+      /**
+       * Handles pointerdown event. This is used for swipe selection process.
+       * to get the start pre_x and pre_y. And also add a pointermove handler
+       * to start handling the swipe selection event.
+       * @param {PointerEvent} event The pointerup event that received by
+       *     kb-keyboard.
+       */
+      down: function(event) {
+        swipeStatus.pre_x = event.screenX;
+        swipeStatus.pre_y = event.screenY;
+        this.addEventListener("pointermove", this.swipeHandler, false);
+      },
+
+      /**
        * Handles pointerup event. This is used for double tap/click events.
        * @param {PointerEvent} event The pointerup event that bubbled to
        *     kb-keyboard.
        */
       up: function(event) {
-        if(this.dblDetail_) {
+        if (this.dblDetail_) {
           this.dblDetail_.clickCount++;
           if (this.dblDetail_.clickCount == 2) {
             this.keyset = this.dblDetail_.toKeyset;
@@ -188,6 +337,18 @@
             this.dblDetail_ = null;
           }
         }
+
+        // TODO(zyaozhujun): There are some edge cases to deal with later.
+        // (for instance, what if a second finger trigger a down and up
+	// event sequence while swiping).
+        // When pointer up from the screen, a swipe selection session finished,
+        // all the data should be reset to prepare for the next session.
+        if (swipeInProgress) {
+          swipeInProgress = false;
+          swipeStatus.resetAll();
+        }
+        // Remove the pointermove event hander here.
+        this.removeEventListener('pointermove', this.swipeHandler, false);
       },
 
       /**
@@ -198,7 +359,8 @@
       keyUp: function(event, detail) {
         if (this.skipEvent(detail))
           return;
-
+        if (swipeInProgress)
+          return;
         this.lastPressedKey.classList.remove('active');
         if (this.lastPressedKey != event.target)
           return;
@@ -226,6 +388,7 @@
         switch(char) {
           case 'Invalid':
           case 'Shift':
+            swipeStatus.swipeFlags = 0;
             return;
           case 'Microphone':
             this.voiceInput_.onDown();
@@ -273,7 +436,7 @@
             this.selectDefaultKeyset();
           } else {
             console.error('Unable to find layout ' + this.layout);
-          }
+         }
         }
       },