diff options
author | satish@chromium.org <satish@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2011-04-05 06:11:06 +0000 |
---|---|---|
committer | satish@chromium.org <satish@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2011-04-05 06:11:06 +0000 |
commit | 4038d2dcf68d300f44032f98f9323c341cef4333 (patch) | |
tree | a301fc5106d4849ac31b2f607909b3d51b5a48ca | |
parent | 189b01cd199726bd100fcd66fcbaee4f898c04a4 (diff) | |
download | chromium_src-4038d2dcf68d300f44032f98f9323c341cef4333.zip chromium_src-4038d2dcf68d300f44032f98f9323c341cef4333.tar.gz chromium_src-4038d2dcf68d300f44032f98f9323c341cef4333.tar.bz2 |
In speech input, introduce a 'warm up' screen for slow audio capture devices.
In some hardware/OS configurations it takes several hundred milliseconds to seconds for the audio driver
to get initialised and start giving audio to the application. We start by showing an empty bubble and if
audio was received within 500ms the usual volume meter becomes visible. If audio data was not received
within 500ms we start showing a light gray spinner animation to signal that things aren't ready yet to
start speaking.
BUG=61677
TEST=unit_tests and browser_tests with --gtest_filter=Speech*
Review URL: http://codereview.chromium.org/6720031
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@80417 0039d316-1c4b-4281-b951-d872f2087c98
15 files changed, 231 insertions, 90 deletions
diff --git a/chrome/browser/speech/speech_input_bubble.cc b/chrome/browser/speech/speech_input_bubble.cc index 93330e0..9537476 100644 --- a/chrome/browser/speech/speech_input_bubble.cc +++ b/chrome/browser/speech/speech_input_bubble.cc @@ -9,17 +9,27 @@ #include "ui/base/resource/resource_bundle.h" #include "ui/gfx/canvas_skia.h" #include "ui/gfx/rect.h" +#include "ui/gfx/skbitmap_operations.h" + +namespace { + +color_utils::HSL kGrayscaleShift = { -1, 0, 0.6 }; + +SkBitmap* mic_full_ = NULL; // Mic image with full volume. +SkBitmap* mic_noise_ = NULL; // Mic image with full noise volume. +SkBitmap* mic_empty_ = NULL; // Mic image with zero volume. +SkBitmap* mic_mask_ = NULL; // Gradient mask used by the volume indicator. +SkBitmap* spinner_ = NULL; // Spinner image for the progress animation. + +const int kWarmingUpAnimationStartMs = 500; +const int kWarmingUpAnimationStepMs = 100; +const int kRecognizingAnimationStepMs = 100; + +} // namespace SpeechInputBubble::FactoryMethod SpeechInputBubble::factory_ = NULL; const int SpeechInputBubble::kBubbleTargetOffsetX = 10; -SkBitmap* SpeechInputBubbleBase::mic_empty_ = NULL; -SkBitmap* SpeechInputBubbleBase::mic_noise_ = NULL; -SkBitmap* SpeechInputBubbleBase::mic_full_ = NULL; -SkBitmap* SpeechInputBubbleBase::mic_mask_ = NULL; -SkBitmap* SpeechInputBubbleBase::spinner_ = NULL; -const int SpeechInputBubbleBase::kRecognizingAnimationStepMs = 100; - SpeechInputBubble* SpeechInputBubble::Create(TabContents* tab_contents, Delegate* delegate, const gfx::Rect& element_rect) { @@ -65,6 +75,22 @@ SpeechInputBubbleBase::SpeechInputBubbleBase(TabContents* tab_contents) // horizontal/wide image. Each animation frame is square in shape within the // sprite. const int kFrameSize = spinner_->height(); + + // When recording starts up, it may take a short while (few ms or even a + // couple of seconds) before the audio device starts really capturing data. + // This is more apparent on first use. To cover such cases we show a warming + // up state in the bubble starting with a blank spinner image. If audio data + // starts coming in within a couple hundred ms, we switch to the recording + // UI and if it takes longer, we show the real warm up animation frames. + // This reduces visual jank for the most part. + // TODO(satish): Change this to create the frames only once on first use + // instead of keeping them as instance variables in every bubble. + SkBitmap empty_spinner; + empty_spinner.setConfig(SkBitmap::kARGB_8888_Config, kFrameSize, kFrameSize); + empty_spinner.allocPixels(); + empty_spinner.eraseRGB(255, 255, 255); + warming_up_frames_.push_back(empty_spinner); + for (SkIRect src_rect(SkIRect::MakeWH(kFrameSize, kFrameSize)); src_rect.fLeft < spinner_->width(); src_rect.offset(kFrameSize, 0)) { @@ -79,6 +105,10 @@ SpeechInputBubbleBase::SpeechInputBubbleBase(TabContents* tab_contents) SkBitmap frame_copy; frame.copyTo(&frame_copy, SkBitmap::kARGB_8888_Config); animation_frames_.push_back(frame_copy); + + // The warm up spinner animation is a gray scale version of the real one. + warming_up_frames_.push_back(SkBitmapOperations::CreateHSLShiftedBitmap( + frame_copy, kGrayscaleShift)); } } @@ -88,9 +118,30 @@ SpeechInputBubbleBase::~SpeechInputBubbleBase() { // member variables which they don't use. } +void SpeechInputBubbleBase::SetWarmUpMode() { + task_factory_.RevokeAll(); + display_mode_ = DISPLAY_MODE_WARM_UP; + animation_step_ = 0; + DoWarmingUpAnimationStep(); + UpdateLayout(); +} + +void SpeechInputBubbleBase::DoWarmingUpAnimationStep() { + SetImage(warming_up_frames_[animation_step_]); + MessageLoop::current()->PostDelayedTask( + FROM_HERE, + task_factory_.NewRunnableMethod( + &SpeechInputBubbleBase::DoWarmingUpAnimationStep), + animation_step_ == 0 ? kWarmingUpAnimationStartMs + : kWarmingUpAnimationStepMs); + if (++animation_step_ >= static_cast<int>(animation_frames_.size())) + animation_step_ = 1; // Frame 0 is skipped during the animation. +} + void SpeechInputBubbleBase::SetRecordingMode() { task_factory_.RevokeAll(); display_mode_ = DISPLAY_MODE_RECORDING; + SetInputVolume(0, 0); UpdateLayout(); } @@ -159,3 +210,12 @@ void SpeechInputBubbleBase::SetInputVolume(float volume, float noise_volume) { TabContents* SpeechInputBubbleBase::tab_contents() { return tab_contents_; } + +void SpeechInputBubbleBase::SetImage(const SkBitmap& image) { + icon_image_.reset(new SkBitmap(image)); + UpdateImage(); +} + +SkBitmap SpeechInputBubbleBase::icon_image() { + return (icon_image_ != NULL) ? *icon_image_ : SkBitmap(); +} diff --git a/chrome/browser/speech/speech_input_bubble.h b/chrome/browser/speech/speech_input_bubble.h index 760da78..9ae4d47 100644 --- a/chrome/browser/speech/speech_input_bubble.h +++ b/chrome/browser/speech/speech_input_bubble.h @@ -79,6 +79,10 @@ class SpeechInputBubble { virtual ~SpeechInputBubble() {} + // Indicates to the user that audio hardware is initializing. If the bubble is + // hidden, |Show| must be called to make it appear on screen. + virtual void SetWarmUpMode() = 0; + // Indicates to the user that audio recording is in progress. If the bubble is // hidden, |Show| must be called to make it appear on screen. virtual void SetRecordingMode() = 0; @@ -119,6 +123,7 @@ class SpeechInputBubbleBase : public SpeechInputBubble { // The current display mode of the bubble, useful only for the platform // specific implementation. enum DisplayMode { + DISPLAY_MODE_WARM_UP, DISPLAY_MODE_RECORDING, DISPLAY_MODE_RECOGNIZING, DISPLAY_MODE_MESSAGE @@ -128,6 +133,7 @@ class SpeechInputBubbleBase : public SpeechInputBubble { virtual ~SpeechInputBubbleBase(); // SpeechInputBubble methods + virtual void SetWarmUpMode(); virtual void SetRecordingMode(); virtual void SetRecognizingMode(); virtual void SetMessage(const string16& text); @@ -138,10 +144,8 @@ class SpeechInputBubbleBase : public SpeechInputBubble { // Updates the platform specific UI layout for the current display mode. virtual void UpdateLayout() = 0; - // Sets the given image as the image to display in the speech bubble. - // TODO(satish): Make the SetRecognizingMode call use this to show an - // animation while waiting for results. - virtual void SetImage(const SkBitmap& image) = 0; + // Overridden by subclasses to copy |icon_image()| to the screen. + virtual void UpdateImage() = 0; DisplayMode display_mode() { return display_mode_; @@ -151,8 +155,12 @@ class SpeechInputBubbleBase : public SpeechInputBubble { return message_text_; } + SkBitmap icon_image(); + private: void DoRecognizingAnimationStep(); + void DoWarmingUpAnimationStep(); + void SetImage(const SkBitmap& image); void DrawVolumeOverlay(SkCanvas* canvas, const SkBitmap& bitmap, @@ -162,6 +170,7 @@ class SpeechInputBubbleBase : public SpeechInputBubble { ScopedRunnableMethodFactory<SpeechInputBubbleBase> task_factory_; int animation_step_; // Current index/step of the animation. std::vector<SkBitmap> animation_frames_; + std::vector<SkBitmap> warming_up_frames_; DisplayMode display_mode_; string16 message_text_; // Text displayed in DISPLAY_MODE_MESSAGE @@ -171,13 +180,8 @@ class SpeechInputBubbleBase : public SpeechInputBubble { scoped_ptr<SkBitmap> buffer_image_; // TabContents in which this this bubble gets displayed. TabContents* tab_contents_; - - static SkBitmap* mic_full_; // Mic image with full volume. - static SkBitmap* mic_noise_; // Mic image with full noise volume. - static SkBitmap* mic_empty_; // Mic image with zero volume. - static SkBitmap* mic_mask_; // Gradient mask used by the volume indicator. - static SkBitmap* spinner_; // Spinner image for the progress animation. - static const int kRecognizingAnimationStepMs; + // The current image displayed in the bubble's icon widget. + scoped_ptr<SkBitmap> icon_image_; }; // This typedef is to workaround the issue with certain versions of diff --git a/chrome/browser/speech/speech_input_bubble_controller.cc b/chrome/browser/speech/speech_input_bubble_controller.cc index 8f0b23c..56ea3f0 100644 --- a/chrome/browser/speech/speech_input_bubble_controller.cc +++ b/chrome/browser/speech/speech_input_bubble_controller.cc @@ -55,6 +55,11 @@ void SpeechInputBubbleController::CloseBubble(int caller_id) { ProcessRequestInUiThread(caller_id, REQUEST_CLOSE, string16(), 0, 0); } +void SpeechInputBubbleController::SetBubbleWarmUpMode(int caller_id) { + ProcessRequestInUiThread(caller_id, REQUEST_SET_WARM_UP_MODE, + string16(), 0, 0); +} + void SpeechInputBubbleController::SetBubbleRecordingMode(int caller_id) { ProcessRequestInUiThread(caller_id, REQUEST_SET_RECORDING_MODE, string16(), 0, 0); @@ -146,7 +151,7 @@ void SpeechInputBubbleController::ProcessRequestInUiThread( if (!bubbles_.count(caller_id)) return; - bool change_active_bubble = (type == REQUEST_SET_RECORDING_MODE || + bool change_active_bubble = (type == REQUEST_SET_WARM_UP_MODE || type == REQUEST_SET_MESSAGE); if (change_active_bubble) { if (current_bubble_caller_id_ && current_bubble_caller_id_ != caller_id) @@ -156,6 +161,9 @@ void SpeechInputBubbleController::ProcessRequestInUiThread( SpeechInputBubble* bubble = bubbles_[caller_id]; switch (type) { + case REQUEST_SET_WARM_UP_MODE: + bubble->SetWarmUpMode(); + break; case REQUEST_SET_RECORDING_MODE: bubble->SetRecordingMode(); break; diff --git a/chrome/browser/speech/speech_input_bubble_controller.h b/chrome/browser/speech/speech_input_bubble_controller.h index c690fae..c09f184 100644 --- a/chrome/browser/speech/speech_input_bubble_controller.h +++ b/chrome/browser/speech/speech_input_bubble_controller.h @@ -55,6 +55,10 @@ class SpeechInputBubbleController int render_view_id, const gfx::Rect& element_rect); + // Indicates to the user that audio hardware is warming up. This also makes + // the bubble visible if not already visible. + void SetBubbleWarmUpMode(int caller_id); + // Indicates to the user that audio recording is in progress. This also makes // the bubble visible if not already visible. void SetBubbleRecordingMode(int caller_id); @@ -84,6 +88,7 @@ class SpeechInputBubbleController private: // The various calls received by this object and handled in the UI thread. enum RequestType { + REQUEST_SET_WARM_UP_MODE, REQUEST_SET_RECORDING_MODE, REQUEST_SET_RECOGNIZING_MODE, REQUEST_SET_MESSAGE, @@ -116,7 +121,7 @@ class SpeechInputBubbleController // Only accessed in the IO thread. Delegate* delegate_; - //*** The following are accessed only in the UI thread. + // *** The following are accessed only in the UI thread. // The caller id for currently visible bubble (since only one bubble is // visible at any time). diff --git a/chrome/browser/speech/speech_input_bubble_controller_unittest.cc b/chrome/browser/speech/speech_input_bubble_controller_unittest.cc index 3ea02b7..a24e0c9 100644 --- a/chrome/browser/speech/speech_input_bubble_controller_unittest.cc +++ b/chrome/browser/speech/speech_input_bubble_controller_unittest.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2010 The Chromium Authors. All rights reserved. +// Copyright (c) 2011 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -59,7 +59,7 @@ class MockSpeechInputBubble : public SpeechInputBubbleBase { virtual void Show() {} virtual void Hide() {} virtual void UpdateLayout() {} - virtual void SetImage(const SkBitmap&) {} + virtual void UpdateImage() {} private: static BubbleType type_; @@ -123,7 +123,7 @@ class SpeechInputBubbleControllerTest static void ActivateBubble() { if (MockSpeechInputBubble::type() == MockSpeechInputBubble::BUBBLE_TEST_FOCUS_CHANGED) { - test_fixture_->controller_->SetBubbleRecordingMode(kBubbleCallerId); + test_fixture_->controller_->SetBubbleWarmUpMode(kBubbleCallerId); } else { test_fixture_->controller_->SetBubbleMessage(kBubbleCallerId, ASCIIToUTF16("Test")); diff --git a/chrome/browser/speech/speech_input_bubble_gtk.cc b/chrome/browser/speech/speech_input_bubble_gtk.cc index ec82451..06bdbb0 100644 --- a/chrome/browser/speech/speech_input_bubble_gtk.cc +++ b/chrome/browser/speech/speech_input_bubble_gtk.cc @@ -49,7 +49,7 @@ class SpeechInputBubbleGtk virtual void Show(); virtual void Hide(); virtual void UpdateLayout(); - virtual void SetImage(const SkBitmap& image); + virtual void UpdateImage(); CHROMEGTK_CALLBACK_0(SpeechInputBubbleGtk, void, OnCancelClicked); CHROMEGTK_CALLBACK_0(SpeechInputBubbleGtk, void, OnTryAgainClicked); @@ -61,6 +61,7 @@ class SpeechInputBubbleGtk bool did_invoke_close_; GtkWidget* label_; + GtkWidget* cancel_button_; GtkWidget* try_again_button_; GtkWidget* icon_; GtkWidget* mic_settings_; @@ -77,6 +78,7 @@ SpeechInputBubbleGtk::SpeechInputBubbleGtk(TabContents* tab_contents, element_rect_(element_rect), did_invoke_close_(false), label_(NULL), + cancel_button_(NULL), try_again_button_(NULL), icon_(NULL), mic_settings_(NULL) { @@ -146,10 +148,10 @@ void SpeechInputBubbleGtk::Show() { gtk_box_pack_start(GTK_BOX(vbox), button_bar, FALSE, FALSE, kBubbleControlVerticalSpacing); - GtkWidget* cancel_button = gtk_button_new_with_label( + cancel_button_ = gtk_button_new_with_label( l10n_util::GetStringUTF8(IDS_CANCEL).c_str()); - gtk_box_pack_start(GTK_BOX(button_bar), cancel_button, TRUE, FALSE, 0); - g_signal_connect(cancel_button, "clicked", + gtk_box_pack_start(GTK_BOX(button_bar), cancel_button_, TRUE, FALSE, 0); + g_signal_connect(cancel_button_, "clicked", G_CALLBACK(&OnCancelClickedThunk), this); try_again_button_ = gtk_button_new_with_label( @@ -206,23 +208,25 @@ void SpeechInputBubbleGtk::UpdateLayout() { if (display_mode() == DISPLAY_MODE_RECORDING) { gtk_label_set_text(GTK_LABEL(label_), l10n_util::GetStringUTF8(IDS_SPEECH_INPUT_BUBBLE_HEADING).c_str()); - SkBitmap* image = ResourceBundle::GetSharedInstance().GetBitmapNamed( - IDR_SPEECH_INPUT_MIC_EMPTY); - GdkPixbuf* pixbuf = gfx::GdkPixbufFromSkBitmap(image); - gtk_image_set_from_pixbuf(GTK_IMAGE(icon_), pixbuf); - g_object_unref(pixbuf); gtk_widget_show(label_); } else { gtk_widget_hide(label_); } + UpdateImage(); gtk_widget_show(icon_); gtk_widget_hide(try_again_button_); if (mic_settings_) gtk_widget_hide(mic_settings_); + if (display_mode() == DISPLAY_MODE_WARM_UP) { + gtk_widget_hide(cancel_button_); + } else { + gtk_widget_show(cancel_button_); + } } } -void SpeechInputBubbleGtk::SetImage(const SkBitmap& image) { +void SpeechInputBubbleGtk::UpdateImage() { + SkBitmap image = icon_image(); if (image.isNull() || !info_bubble_) return; diff --git a/chrome/browser/speech/speech_input_bubble_mac.mm b/chrome/browser/speech/speech_input_bubble_mac.mm index f05c7cf..0b3ea16 100644 --- a/chrome/browser/speech/speech_input_bubble_mac.mm +++ b/chrome/browser/speech/speech_input_bubble_mac.mm @@ -26,7 +26,7 @@ class SpeechInputBubbleImpl : public SpeechInputBubbleBase { virtual void Show(); virtual void Hide(); virtual void UpdateLayout(); - virtual void SetImage(const SkBitmap& image); + virtual void UpdateImage(); private: scoped_nsobject<SpeechInputWindowController> window_; @@ -47,9 +47,9 @@ SpeechInputBubbleImpl::~SpeechInputBubbleImpl() { [window_.get() close]; } -void SpeechInputBubbleImpl::SetImage(const SkBitmap& image) { +void SpeechInputBubbleImpl::UpdateImage() { if (window_.get()) - [window_.get() setImage:gfx::SkBitmapToNSImage(image)]; + [window_.get() setImage:gfx::SkBitmapToNSImage(icon_image())]; } void SpeechInputBubbleImpl::Show() { @@ -93,7 +93,8 @@ void SpeechInputBubbleImpl::UpdateLayout() { return; [window_.get() updateLayout:display_mode() - messageText:message_text()]; + messageText:message_text() + iconImage:gfx::SkBitmapToNSImage(icon_image())]; } } // namespace diff --git a/chrome/browser/speech/speech_input_bubble_views.cc b/chrome/browser/speech/speech_input_bubble_views.cc index e1906c2e..52c38e1 100644 --- a/chrome/browser/speech/speech_input_bubble_views.cc +++ b/chrome/browser/speech/speech_input_bubble_views.cc @@ -4,6 +4,8 @@ #include "chrome/browser/speech/speech_input_bubble.h" +#include <algorithm> + #include "base/message_loop.h" #include "base/utf_string_conversions.h" #include "chrome/browser/browser_window.h" @@ -39,7 +41,8 @@ class ContentView explicit ContentView(SpeechInputBubbleDelegate* delegate); void UpdateLayout(SpeechInputBubbleBase::DisplayMode mode, - const string16& message_text); + const string16& message_text, + const SkBitmap& image); void SetImage(const SkBitmap& image); // views::ButtonListener methods. @@ -60,12 +63,17 @@ class ContentView views::NativeButton* try_again_; views::NativeButton* cancel_; views::Link* mic_settings_; + SpeechInputBubbleBase::DisplayMode display_mode_; + const int kIconLayoutMinWidth; DISALLOW_COPY_AND_ASSIGN(ContentView); }; ContentView::ContentView(SpeechInputBubbleDelegate* delegate) - : delegate_(delegate) { + : delegate_(delegate), + display_mode_(SpeechInputBubbleBase::DISPLAY_MODE_WARM_UP), + kIconLayoutMinWidth(ResourceBundle::GetSharedInstance().GetBitmapNamed( + IDR_SPEECH_INPUT_MIC_EMPTY)->width()) { ResourceBundle& rb = ResourceBundle::GetSharedInstance(); const gfx::Font& font = rb.GetFont(ResourceBundle::MediumFont); @@ -86,8 +94,6 @@ ContentView::ContentView(SpeechInputBubbleDelegate* delegate) AddChildView(message_); icon_ = new views::ImageView(); - icon_->SetImage(*ResourceBundle::GetSharedInstance().GetBitmapNamed( - IDR_SPEECH_INPUT_MIC_EMPTY)); icon_->SetHorizontalAlignment(views::ImageView::CENTER); AddChildView(icon_); @@ -108,23 +114,31 @@ ContentView::ContentView(SpeechInputBubbleDelegate* delegate) } void ContentView::UpdateLayout(SpeechInputBubbleBase::DisplayMode mode, - const string16& message_text) { + const string16& message_text, + const SkBitmap& image) { + display_mode_ = mode; bool is_message = (mode == SpeechInputBubbleBase::DISPLAY_MODE_MESSAGE); icon_->SetVisible(!is_message); message_->SetVisible(is_message); mic_settings_->SetVisible(is_message); try_again_->SetVisible(is_message); + cancel_->SetVisible(mode != SpeechInputBubbleBase::DISPLAY_MODE_WARM_UP); heading_->SetVisible(mode == SpeechInputBubbleBase::DISPLAY_MODE_RECORDING); - if (mode == SpeechInputBubbleBase::DISPLAY_MODE_MESSAGE) { + if (is_message) { message_->SetText(UTF16ToWideHack(message_text)); - } else if (mode == SpeechInputBubbleBase::DISPLAY_MODE_RECORDING) { - icon_->SetImage(*ResourceBundle::GetSharedInstance().GetBitmapNamed( - IDR_SPEECH_INPUT_MIC_EMPTY)); + } else { + SetImage(image); } if (icon_->IsVisible()) icon_->ResetImageSize(); + + // When moving from warming up to recording state, the size of the content + // stays the same. So we wouldn't get a resize/layout call from the view + // system and we do it ourselves. + if (GetPreferredSize() == size()) // |size()| here is the current size. + Layout(); } void ContentView::SetImage(const SkBitmap& image) { @@ -154,15 +168,13 @@ gfx::Size ContentView::GetPreferredSize() { control_width += try_again_->GetPreferredSize().width() + views::kRelatedButtonHSpacing; } - if (control_width > width) - width = control_width; - control_width = icon_->GetPreferredSize().width(); - if (control_width > width) - width = control_width; + width = std::max(width, control_width); + control_width = std::max(icon_->GetPreferredSize().width(), + kIconLayoutMinWidth); + width = std::max(width, control_width); if (mic_settings_->IsVisible()) { control_width = mic_settings_->GetPreferredSize().width(); - if (control_width > width) - width = control_width; + width = std::max(width, control_width); } int height = cancel_->GetPreferredSize().height(); @@ -213,6 +225,8 @@ void ContentView::Layout() { DCHECK(icon_->IsVisible()); int control_height = icon_->GetImage().height(); + if (display_mode_ == SpeechInputBubbleBase::DISPLAY_MODE_WARM_UP) + y = (available_height - control_height) / 2; icon_->SetBounds(x, y, available_width, control_height); y += control_height; @@ -222,10 +236,12 @@ void ContentView::Layout() { y += control_height; } - control_height = cancel_->GetPreferredSize().height(); - int width = cancel_->GetPreferredSize().width(); - cancel_->SetBounds(x + (available_width - width) / 2, y, width, - control_height); + if (cancel_->IsVisible()) { + control_height = cancel_->GetPreferredSize().height(); + int width = cancel_->GetPreferredSize().width(); + cancel_->SetBounds(x + (available_width - width) / 2, y, width, + control_height); + } } } @@ -245,7 +261,7 @@ class SpeechInputBubbleImpl // SpeechInputBubbleBase methods. virtual void UpdateLayout(); - virtual void SetImage(const SkBitmap& image); + virtual void UpdateImage(); // Returns the screen rectangle to use as the info bubble's target. // |element_rect| is the html element's bounds in page coordinates. @@ -344,14 +360,14 @@ void SpeechInputBubbleImpl::Hide() { void SpeechInputBubbleImpl::UpdateLayout() { if (bubble_content_) - bubble_content_->UpdateLayout(display_mode(), message_text()); + bubble_content_->UpdateLayout(display_mode(), message_text(), icon_image()); if (info_bubble_) // Will be null on first call. info_bubble_->SizeToContents(); } -void SpeechInputBubbleImpl::SetImage(const SkBitmap& image) { +void SpeechInputBubbleImpl::UpdateImage() { if (bubble_content_) - bubble_content_->SetImage(image); + bubble_content_->SetImage(icon_image()); } } // namespace diff --git a/chrome/browser/speech/speech_input_manager.cc b/chrome/browser/speech/speech_input_manager.cc index 56336b9..fb06dd5d 100644 --- a/chrome/browser/speech/speech_input_manager.cc +++ b/chrome/browser/speech/speech_input_manager.cc @@ -111,6 +111,7 @@ class SpeechInputManagerImpl : public SpeechInputManager, SpeechInputManagerDelegate* delegate); // SpeechRecognizer::Delegate methods. + virtual void DidStartReceivingAudio(int caller_id); virtual void SetRecognitionResult(int caller_id, bool error, const SpeechInputResultArray& result); @@ -247,7 +248,7 @@ void SpeechInputManagerImpl::StartRecognitionForRequest(int caller_id) { recording_caller_id_ = caller_id; requests_[caller_id].is_active = true; requests_[caller_id].recognizer->StartRecording(); - bubble_controller_->SetBubbleRecordingMode(caller_id); + bubble_controller_->SetBubbleWarmUpMode(caller_id); } } @@ -334,6 +335,12 @@ void SpeechInputManagerImpl::OnRecognizerError( NOTREACHED() << "unknown error " << error; } +void SpeechInputManagerImpl::DidStartReceivingAudio(int caller_id) { + DCHECK(HasPendingRequest(caller_id)); + DCHECK(recording_caller_id_ == caller_id); + bubble_controller_->SetBubbleRecordingMode(caller_id); +} + void SpeechInputManagerImpl::DidCompleteEnvironmentEstimation(int caller_id) { DCHECK(HasPendingRequest(caller_id)); DCHECK(recording_caller_id_ == caller_id); diff --git a/chrome/browser/ui/cocoa/speech_input_window_controller.h b/chrome/browser/ui/cocoa/speech_input_window_controller.h index 4538a83..c472ecc 100644 --- a/chrome/browser/ui/cocoa/speech_input_window_controller.h +++ b/chrome/browser/ui/cocoa/speech_input_window_controller.h @@ -1,4 +1,4 @@ -// Copyright (c) 2010 The Chromium Authors. All rights reserved. +// Copyright (c) 2011 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -16,6 +16,7 @@ @interface SpeechInputWindowController : BaseBubbleController { @private SpeechInputBubble::Delegate* delegate_; // weak. + SpeechInputBubbleBase::DisplayMode displayMode_; // References below are weak, being obtained from the nib. IBOutlet NSImageView* iconImage_; @@ -41,7 +42,8 @@ // Updates the UI with data related to the given display mode. - (void)updateLayout:(SpeechInputBubbleBase::DisplayMode)mode - messageText:(const string16&)messageText; + messageText:(const string16&)messageText + iconImage:(NSImage*)iconImage; // Makes the speech input bubble visible on screen. - (void)show; diff --git a/chrome/browser/ui/cocoa/speech_input_window_controller.mm b/chrome/browser/ui/cocoa/speech_input_window_controller.mm index 105e84c..0f966f0 100644 --- a/chrome/browser/ui/cocoa/speech_input_window_controller.mm +++ b/chrome/browser/ui/cocoa/speech_input_window_controller.mm @@ -1,4 +1,4 @@ -// Copyright (c) 2010 The Chromium Authors. All rights reserved. +// Copyright (c) 2011 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -36,6 +36,7 @@ const int kInstructionLabelMaxWidth = 150; anchoredAt:anchoredAt])) { DCHECK(delegate); delegate_ = delegate; + displayMode_ = SpeechInputBubbleBase::DISPLAY_MODE_WARM_UP; } return self; } @@ -72,13 +73,25 @@ const int kInstructionLabelMaxWidth = 150; if (![tryAgainButton_ isHidden]) newWidth += tryAgainSize.width; + // The size of the bubble in warm up mode is fixed to be the same as in + // recording mode, so from warm up it can transition to recording without any + // UI jank. + bool isWarmUp = (displayMode_ == + SpeechInputBubbleBase::DISPLAY_MODE_WARM_UP); + if (![iconImage_ isHidden]) { NSSize size = [[iconImage_ image] size]; + if (isWarmUp) { + NSImage* volumeIcon = + ResourceBundle::GetSharedInstance().GetNativeImageNamed( + IDR_SPEECH_INPUT_MIC_EMPTY); + size = [volumeIcon size]; + } newHeight += size.height; newWidth = std::max(newWidth, size.width + 2 * kBubbleHorizontalMargin); } - if (![instructionLabel_ isHidden]) { + if (![instructionLabel_ isHidden] || isWarmUp) { [instructionLabel_ sizeToFit]; NSSize textSize = [[instructionLabel_ cell] cellSize]; NSRect boundsRect = NSMakeRect(0, 0, kInstructionLabelMaxWidth, @@ -118,9 +131,11 @@ const int kInstructionLabelMaxWidth = 150; [tryAgainButton_ setFrame:tryAgainRect]; } cancelRect.origin.y = y; - [cancelButton_ setFrame:cancelRect]; - y += NSHeight(cancelRect) + kBubbleControlVerticalSpacing; + if (![cancelButton_ isHidden]) { + [cancelButton_ setFrame:cancelRect]; + y += NSHeight(cancelRect) + kBubbleControlVerticalSpacing; + } NSRect rect; if (![micSettingsButton_ isHidden]) { @@ -146,6 +161,9 @@ const int kInstructionLabelMaxWidth = 150; if (![iconImage_ isHidden]) { rect.size = [[iconImage_ image] size]; + // In warm-up mode only the icon gets displayed so center it vertically. + if (displayMode_ == SpeechInputBubbleBase::DISPLAY_MODE_WARM_UP) + y = (size.height - rect.size.height) / 2; rect.origin.x = (size.width - NSWidth(rect)) / 2; rect.origin.y = y; [iconImage_ setFrame:rect]; @@ -153,34 +171,29 @@ const int kInstructionLabelMaxWidth = 150; } - (void)updateLayout:(SpeechInputBubbleBase::DisplayMode)mode - messageText:(const string16&)messageText { + messageText:(const string16&)messageText + iconImage:(NSImage*)iconImage { // The very first time this method is called, the child views would still be // uninitialized and null. So we invoke [self window] first and that sets up // the child views properly so we can do the layout calculations below. NSWindow* window = [self window]; + displayMode_ = mode; + BOOL is_message = (mode == SpeechInputBubbleBase::DISPLAY_MODE_MESSAGE); + BOOL is_recording = (mode == SpeechInputBubbleBase::DISPLAY_MODE_RECORDING); + BOOL is_warm_up = (mode == SpeechInputBubbleBase::DISPLAY_MODE_WARM_UP); + [iconImage_ setHidden:is_message]; + [tryAgainButton_ setHidden:!is_message]; + [micSettingsButton_ setHidden:!is_message]; + [instructionLabel_ setHidden:!is_message && !is_recording]; + [cancelButton_ setHidden:is_warm_up]; // Get the right set of controls to be visible. - if (mode == SpeechInputBubbleBase::DISPLAY_MODE_MESSAGE) { + if (is_message) { [instructionLabel_ setStringValue:base::SysUTF16ToNSString(messageText)]; - [iconImage_ setHidden:YES]; - [tryAgainButton_ setHidden:NO]; - [instructionLabel_ setHidden:NO]; - [micSettingsButton_ setHidden:NO]; } else { - if (mode == SpeechInputBubbleBase::DISPLAY_MODE_RECORDING) { - [instructionLabel_ setStringValue:l10n_util::GetNSString( - IDS_SPEECH_INPUT_BUBBLE_HEADING)]; - [instructionLabel_ setHidden:NO]; - NSImage* icon = ResourceBundle::GetSharedInstance().GetNativeImageNamed( - IDR_SPEECH_INPUT_MIC_EMPTY); - [iconImage_ setImage:icon]; - } else { - [instructionLabel_ setHidden:YES]; - } - [iconImage_ setHidden:NO]; - [iconImage_ setNeedsDisplay:YES]; - [tryAgainButton_ setHidden:YES]; - [micSettingsButton_ setHidden:YES]; + [iconImage_ setImage:iconImage]; + [instructionLabel_ setStringValue:l10n_util::GetNSString( + IDS_SPEECH_INPUT_BUBBLE_HEADING)]; } NSSize newSize = [self calculateContentSize]; diff --git a/content/browser/speech/speech_recognition_request.cc b/content/browser/speech/speech_recognition_request.cc index 7d66aa7..19d8204 100644 --- a/content/browser/speech/speech_recognition_request.cc +++ b/content/browser/speech/speech_recognition_request.cc @@ -27,7 +27,7 @@ const char* const kConfidenceString = "confidence"; // TODO(satish): Remove this hardcoded value once the page is allowed to // set this via an attribute. -const int kMaxResults = 5; +const int kMaxResults = 6; bool ParseServerResponse(const std::string& response_body, speech_input::SpeechInputResultArray* result) { diff --git a/content/browser/speech/speech_recognizer.cc b/content/browser/speech/speech_recognizer.cc index 1807db0..7a4ebd1 100644 --- a/content/browser/speech/speech_recognizer.cc +++ b/content/browser/speech/speech_recognizer.cc @@ -219,7 +219,8 @@ void SpeechRecognizer::HandleOnData(string* data) { if (request_ == NULL) { // This was the first audio packet recorded, so start a request to the - // server to send the data. + // server to send the data and inform the delegate. + delegate_->DidStartReceivingAudio(caller_id_); request_.reset(new SpeechRecognitionRequest( Profile::GetDefaultRequestContext(), this)); request_->Start(language_, grammar_, hardware_info_, origin_url_, diff --git a/content/browser/speech/speech_recognizer.h b/content/browser/speech/speech_recognizer.h index 1afdecb..a87831e 100644 --- a/content/browser/speech/speech_recognizer.h +++ b/content/browser/speech/speech_recognizer.h @@ -41,6 +41,10 @@ class SpeechRecognizer bool error, const SpeechInputResultArray& result) = 0; + // Invoked when the first audio packet was received from the audio capture + // device. + virtual void DidStartReceivingAudio(int caller_id) = 0; + // Invoked when audio recording stops, either due to the end pointer // detecting silence in user input or if |StopRecording| was called. The // delegate has to wait until |DidCompleteRecognition| is invoked before diff --git a/content/browser/speech/speech_recognizer_unittest.cc b/content/browser/speech/speech_recognizer_unittest.cc index 738eaf9..7239582 100644 --- a/content/browser/speech/speech_recognizer_unittest.cc +++ b/content/browser/speech/speech_recognizer_unittest.cc @@ -30,6 +30,7 @@ class SpeechRecognizerTest : public SpeechRecognizerDelegate, recording_complete_(false), recognition_complete_(false), result_received_(false), + audio_received_(false), error_(SpeechRecognizer::RECOGNIZER_NO_ERROR), volume_(-1.0f) { int audio_packet_length_bytes = @@ -58,6 +59,10 @@ class SpeechRecognizerTest : public SpeechRecognizerDelegate, virtual void DidCompleteEnvironmentEstimation(int caller_id) { } + virtual void DidStartReceivingAudio(int caller_id) { + audio_received_ = true; + } + virtual void OnRecognizerError(int caller_id, SpeechRecognizer::ErrorCode error) { error_ = error; @@ -101,6 +106,7 @@ class SpeechRecognizerTest : public SpeechRecognizerDelegate, bool recording_complete_; bool recognition_complete_; bool result_received_; + bool audio_received_; SpeechRecognizer::ErrorCode error_; TestURLFetcherFactory url_fetcher_factory_; TestAudioInputControllerFactory audio_input_controller_factory_; @@ -116,6 +122,7 @@ TEST_F(SpeechRecognizerTest, StopNoData) { EXPECT_FALSE(recording_complete_); EXPECT_FALSE(recognition_complete_); EXPECT_FALSE(result_received_); + EXPECT_FALSE(audio_received_); EXPECT_EQ(SpeechRecognizer::RECOGNIZER_NO_ERROR, error_); } @@ -127,6 +134,7 @@ TEST_F(SpeechRecognizerTest, CancelNoData) { EXPECT_TRUE(recording_complete_); EXPECT_TRUE(recognition_complete_); EXPECT_FALSE(result_received_); + EXPECT_FALSE(audio_received_); EXPECT_EQ(SpeechRecognizer::RECOGNIZER_NO_ERROR, error_); } @@ -153,6 +161,7 @@ TEST_F(SpeechRecognizerTest, StopWithData) { } recognizer_->StopRecording(); + EXPECT_TRUE(audio_received_); EXPECT_TRUE(recording_complete_); EXPECT_FALSE(recognition_complete_); EXPECT_FALSE(result_received_); @@ -183,6 +192,7 @@ TEST_F(SpeechRecognizerTest, CancelWithData) { MessageLoop::current()->RunAllPending(); recognizer_->CancelRecognition(); ASSERT_TRUE(url_fetcher_factory_.GetFetcherByID(0)); + EXPECT_TRUE(audio_received_); EXPECT_FALSE(recording_complete_); EXPECT_FALSE(recognition_complete_); EXPECT_FALSE(result_received_); @@ -203,6 +213,7 @@ TEST_F(SpeechRecognizerTest, ConnectionError) { ASSERT_TRUE(fetcher); recognizer_->StopRecording(); + EXPECT_TRUE(audio_received_); EXPECT_TRUE(recording_complete_); EXPECT_FALSE(recognition_complete_); EXPECT_FALSE(result_received_); @@ -233,6 +244,7 @@ TEST_F(SpeechRecognizerTest, ServerError) { ASSERT_TRUE(fetcher); recognizer_->StopRecording(); + EXPECT_TRUE(audio_received_); EXPECT_TRUE(recording_complete_); EXPECT_FALSE(recognition_complete_); EXPECT_FALSE(result_received_); @@ -257,6 +269,7 @@ TEST_F(SpeechRecognizerTest, AudioControllerErrorNoData) { ASSERT_TRUE(controller); controller->event_handler()->OnError(controller, 0); MessageLoop::current()->RunAllPending(); + EXPECT_FALSE(audio_received_); EXPECT_FALSE(recording_complete_); EXPECT_FALSE(recognition_complete_); EXPECT_FALSE(result_received_); @@ -275,6 +288,7 @@ TEST_F(SpeechRecognizerTest, AudioControllerErrorWithData) { controller->event_handler()->OnError(controller, 0); MessageLoop::current()->RunAllPending(); ASSERT_TRUE(url_fetcher_factory_.GetFetcherByID(0)); + EXPECT_TRUE(audio_received_); EXPECT_FALSE(recording_complete_); EXPECT_FALSE(recognition_complete_); EXPECT_FALSE(result_received_); @@ -299,6 +313,7 @@ TEST_F(SpeechRecognizerTest, NoSpeechCallbackIssued) { audio_packet_.size()); } MessageLoop::current()->RunAllPending(); + EXPECT_TRUE(audio_received_); EXPECT_FALSE(recording_complete_); EXPECT_FALSE(recognition_complete_); EXPECT_FALSE(result_received_); @@ -334,6 +349,7 @@ TEST_F(SpeechRecognizerTest, NoSpeechCallbackNotIssued) { MessageLoop::current()->RunAllPending(); EXPECT_EQ(SpeechRecognizer::RECOGNIZER_NO_ERROR, error_); + EXPECT_TRUE(audio_received_); EXPECT_FALSE(recording_complete_); EXPECT_FALSE(recognition_complete_); recognizer_->CancelRecognition(); |