path: root/chrome/browser/renderer_host/translation_service.cc

// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/renderer_host/translation_service.h"

#include "base/json/json_reader.h"
#include "base/stl_util-inl.h"
#include "chrome/browser/profile.h"
#include "chrome/common/render_messages.h"
#include "net/base/escape.h"

#if defined(GOOGLE_CHROME_BUILD)
#include "chrome/browser/renderer_host/translate/translate_internal.h"
#else
// Defining dummy URLs for unit-tests to pass.
#define TRANSLATE_SERVER_URL "http://disabled"
#define TRANSLATE_SERVER_SECURE_URL "https://disabled"
#endif

namespace {

// The URLs we send translation requests to.
const char kServiceURL[] = TRANSLATE_SERVER_URL;
const char kSecureServiceURL[] = TRANSLATE_SERVER_SECURE_URL;

// The different params used when sending requests to the translate server.
const char kVersionParam[] = "v";
const char kLangPairParam[] = "langpair";
const char kTextParam[] = "q";
const char kClientParam[] = "client";
const char kFormatParam[] = "format";
const char kSSLParam[] = "ssl";
const char kTranslationCountParam[] = "tc";

// Mapping from a locale name to a language code name.
// Locale names not included are translated as is.
struct LocaleToCLDLanguage {
  const char* locale_language;  // Language Chrome locale is in.
  const char* cld_language;     // Language the CLD reports.
};
LocaleToCLDLanguage kLocaleToCLDLanguages[] = {
    { "en-GB", "en" },
    { "en-US", "en" },
    { "es-419", "es" },
    { "pt-BR", "pt" },
    { "pt-PT", "pt" },
};

// The list of languages the Google translation server supports.
const char* kSupportedLanguages[] = {
    "af",     // Afrikaans
    "sq",     // Albanian
    "ar",     // Arabic
    "be",     // Belarusian
    "bg",     // Bulgarian
    "ca",     // Catalan
    "zh-CN",  // Chinese (Simplified)
    "zh-TW",  // Chinese (Traditional)
    "hr",     // Croatian
    "cs",     // Czech
    "da",     // Danish
    "nl",     // Dutch
    "en",     // English
    "et",     // Estonian
    "tl",     // Tagalog (Filipino)
    "fi",     // Finnish
    "fr",     // French
    "gl",     // Galician
    "de",     // German
    "el",     // Greek
    "he",     // Hebrew
    "hi",     // Hindi
    "hu",     // Hungarian
    "is",     // Icelandic
    "id",     // Indonesian
    "it",     // Italian
    "ga",     // Irish
    "ja",     // Japanese
    "ko",     // Korean
    "lv",     // Latvian
    "lt",     // Lithuanian
    "mk",     // Macedonian
    "ms",     // Malay
    "mt",     // Maltese
    "no",     // Norwegian
    "fa",     // Persian
    "pl",     // Polish
    "pt",     // Portuguese
    "ro",     // Romanian
    "ru",     // Russian
    "sr",     // Serbian
    "sk",     // Slovak
    "sl",     // Slovenian
    "es",     // Spanish
    "sw",     // Swahili
    "sv",     // Swedish
    "th",     // Thai
    "tr",     // Turkish
    "uk",     // Ukrainian
    "vi",     // Vietnamese
    "cy",     // Welsh
    "yi",     // Yiddish
};

// The maximum size in bytes after which the server will refuse the request.
const size_t kTextRequestMaxSize = 1024 * 30;

// Delay to wait for before sending a request to the translation server.
const int kSendRequestDelay = 100;

// Task used to send the current pending translation request for a renderer
// after some time has elapsed with no new request from that renderer.
// Note that this task is canceled when TranslationRequest is destroyed, which
// happens when the TranslationService is going away.  So it is OK to have it
// have a pointer to the TranslationService.
class SendTranslationRequestTask : public CancelableTask {
 public:
  SendTranslationRequestTask(TranslationService* translation_service,
                             int renderer_id,
                             bool secure);
  virtual void Run();
  virtual void Cancel();

 private:
  TranslationService* translation_service_;
  int renderer_id_;
  bool secure_;
  bool canceled_;

  DISALLOW_COPY_AND_ASSIGN(SendTranslationRequestTask);
};

}  // namespace

// static
// The string is: '&' + kTextParam + '='.
size_t TranslationService::text_param_length_ = 1 + arraysize(kTextParam) + 1;

// static
base::LazyInstance<std::set<std::string> >
    TranslationService::supported_languages_(base::LINKER_INITIALIZED);

// Contains the information necessary to send a request to the translation
// server.  It is used to group several renderer queries, as to limit the
// load sent to the translation server.
struct TranslationService::TranslationRequest {
  TranslationRequest(int routing_id,
                     int page_id,
                     const std::string& source_lang,
                     const std::string& target_lang,
                     bool secure)
      : routing_id(routing_id),
        page_id(page_id),
        source_lang(source_lang),
        target_lang(target_lang),
        secure(secure),
        send_query_task(NULL) {
    renderer_request_info.reset(new RendererRequestInfoList());
  }

  ~TranslationRequest() {
    if (send_query_task)
      send_query_task->Cancel();
  }

  void Clear() {
    page_id = 0;
    source_lang.clear();
    target_lang.clear();
    query.clear();
    renderer_request_info->clear();
    if (send_query_task) {
      send_query_task->Cancel();
      send_query_task = NULL;
    }
  }

  int routing_id;
  int page_id;
  std::string source_lang;
  std::string target_lang;
  bool secure;
  std::string query;
  // renderer_request_info is a scoped_ptr so that we avoid copying the list
  // when the request is sent.  At that point we only transfer ownership of that
  // list to renderer_request_infos_.
  scoped_ptr<RendererRequestInfoList> renderer_request_info;
  CancelableTask* send_query_task;
};

////////////////////////////////////////////////////////////////////////////////
// SendTranslationRequestTask

SendTranslationRequestTask::SendTranslationRequestTask(
    TranslationService* translation_service,
    int renderer_id,
    bool secure)
    : translation_service_(translation_service),
      renderer_id_(renderer_id),
      secure_(secure),
      canceled_(false) {
}

void SendTranslationRequestTask::Run() {
  if (canceled_)
    return;
  translation_service_->
      SendTranslationRequestForRenderer(renderer_id_, secure_);
}

void SendTranslationRequestTask::Cancel() {
  canceled_ = true;
}

////////////////////////////////////////////////////////////////////////////////
// TranslationService, public:

TranslationService::TranslationService(IPC::Message::Sender* message_sender)
    : message_sender_(message_sender),
      kCRAnchorTagStart(ASCIIToUTF16("<a _CR_TR_ id='")),
      kAnchorTagStart(ASCIIToUTF16("<a ")),
      kClosingAnchorTag(ASCIIToUTF16("</a>")),
      kQuote(ASCIIToUTF16("'")),
      kGreaterThan(ASCIIToUTF16(">")),
      kLessThan(ASCIIToUTF16("<")),
      kQuoteGreaterThan(ASCIIToUTF16("'>")) {
}

TranslationService::~TranslationService() {
  STLDeleteContainerPairSecondPointers(pending_translation_requests_.begin(),
                                       pending_translation_requests_.end());
  STLDeleteContainerPairSecondPointers(
      pending_secure_translation_requests_.begin(),
      pending_secure_translation_requests_.end());
  STLDeleteContainerPairPointers(renderer_request_infos_.begin(),
                                 renderer_request_infos_.end());
}

void TranslationService::Translate(int routing_id,
                                   int page_id,
                                   int work_id,
                                   const TextChunks& text_chunks,
                                   const std::string& source_lang,
                                   const std::string& target_lang,
                                   bool secure) {
  TranslationRequestMap& request_map =
      secure ? pending_secure_translation_requests_ :
               pending_translation_requests_;
  TranslationRequestMap::iterator iter = request_map.find(routing_id);
  TranslationRequest* translation_request = NULL;

  string16 utf16_text = MergeTextChunks(text_chunks);
  std::string text = EscapeUrlEncodedData(UTF16ToUTF8(utf16_text));

  if (iter != request_map.end()) {
    translation_request = iter->second;
    if (page_id != translation_request->page_id) {
      // We are getting a request from a renderer for a different page id.
      // This indicates we navigated away from the page that was being
      // translated.  We should drop the current pending translations.
      translation_request->Clear();
      // Set the new states.
      translation_request->page_id = page_id;
      translation_request->source_lang = source_lang;
      translation_request->target_lang = target_lang;
    } else {
      DCHECK(translation_request->source_lang == source_lang);
      DCHECK(translation_request->target_lang == target_lang);
      // Cancel the pending tasks to send the query.  We'll be posting a new one
      // after we updated the request.
      translation_request->send_query_task->Cancel();
      translation_request->send_query_task = NULL;
      if (translation_request->query.size() + text.size() +
              text_param_length_ >= kTextRequestMaxSize) {
        // The request would be too big with that last addition of text, send
        // the request now. (Single requests too big to be sent in 1 translation
        // request are dealt with below.)
        if (!translation_request->query.empty()) {  // Single requests
          SendRequestToTranslationServer(translation_request);
          // The translation request has been deleted.
          translation_request = NULL;
          iter = request_map.end();
        }
      }
    }
  }

  if (translation_request == NULL) {
    translation_request = new TranslationRequest(routing_id, page_id,
                                                 source_lang, target_lang,
                                                 secure);
    request_map[routing_id] = translation_request;
  }

  AddTextToRequestString(&(translation_request->query), text,
                         source_lang, target_lang, secure);

  translation_request->renderer_request_info->push_back(
      RendererRequestInfo(routing_id, work_id));

  if (translation_request->query.size() > kTextRequestMaxSize) {
    DCHECK(translation_request->renderer_request_info->size() == 1U);
    // This one request is too large for the translation service.
    // TODO(jcampan): we should support such requests by splitting them.
    iter = request_map.find(routing_id);
    DCHECK(iter != request_map.end());
    request_map.erase(iter);
    message_sender_->Send(
        new ViewMsg_TranslateTextReponse(routing_id, work_id, 1, TextChunks()));
    delete translation_request;
    return;
  }

  // Now post the new task that will ensure we'll send the request to the
  // translation server if no renderer requests are received within a
  // reasonable amount of time.
  DCHECK(!translation_request->send_query_task);
  translation_request->send_query_task =
      new SendTranslationRequestTask(this, routing_id, secure);
  MessageLoop::current()->PostDelayedTask(FROM_HERE,
      translation_request->send_query_task, GetSendRequestDelay());
}

void TranslationService::SendTranslationRequestForRenderer(int renderer_id,
                                                           bool secure) {
  TranslationRequestMap& request_map =
      secure ? pending_secure_translation_requests_ :
               pending_translation_requests_;
  TranslationRequestMap::const_iterator iter = request_map.find(renderer_id);
  DCHECK(iter != request_map.end());
  SendRequestToTranslationServer(iter->second);
}

void TranslationService::OnURLFetchComplete(const URLFetcher* source,
                                            const GURL& url,
                                            const URLRequestStatus& status,
                                            int response_code,
                                            const ResponseCookies& cookies,
                                            const std::string& data) {
  if (!status.is_success() || response_code != 200 || data.empty()) {
    TranslationFailed(source);
    return;
  }

  // If the response is a simple string, put it in an array.  (The JSONReader
  // requires an array or map at the root.)
  std::string wrapped_data;
  if (data.size() > 1U && data[0] == '"') {
    wrapped_data.append("[");
    wrapped_data.append(data);
    wrapped_data.append("]");
  }
  scoped_ptr<Value> value(base::JSONReader::Read(
      wrapped_data.empty() ? data : wrapped_data, true));
  if (!value.get()) {
    NOTREACHED() << "Translation server returned invalid JSON response.";
    TranslationFailed(source);
    return;
  }

  // If the request was for a single string, the response is the translated
  // string.
  TextChunksList translated_chunks_list;
  if (value->IsType(Value::TYPE_STRING)) {
    string16 translated_text;
    if (!value->GetAsUTF16(&translated_text)) {
      NOTREACHED();
      TranslationFailed(source);
      return;
    }
    TextChunks translated_text_chunks;
    translated_text_chunks.push_back(translated_text);
    translated_chunks_list.push_back(translated_text_chunks);
  } else {
    if (!value->IsType(Value::TYPE_LIST)) {
      NOTREACHED() << "Translation server returned unexpected JSON response "
          " (not a list).";
      TranslationFailed(source);
      return;
    }
    ListValue* translated_text_list = static_cast<ListValue*>(value.get());
    for (size_t i = 0; i < translated_text_list->GetSize(); ++i) {
      string16 translated_text;
      if (!translated_text_list->GetStringAsUTF16(i, &translated_text)) {
        NOTREACHED() << "Translation server returned unexpected JSON response "
            " (unexpected type in list).";
        TranslationFailed(source);
        return;
      }
      translated_text = UnescapeForHTML(translated_text);
      TranslationService::TextChunks translated_text_chunks;
      TranslationService::SplitIntoTextChunks(translated_text,
                                              &translated_text_chunks);
      translated_chunks_list.push_back(translated_text_chunks);
    }
  }

  // We have successfully extracted all the translated text chunks, send them to
  // the renderer.
  SendResponseToRenderer(source, 0, translated_chunks_list);
}

// static
bool TranslationService::IsTranslationEnabled() {
  return GURL(kServiceURL).host() != "disabled";
}

// static
void TranslationService::GetSupportedLanguages(
    std::vector<std::string>* languages) {
  DCHECK(languages && languages->empty());
  for (size_t i = 0; i < arraysize(kSupportedLanguages); ++i)
    languages->push_back(kSupportedLanguages[i]);
}

// static
std::string TranslationService::GetLanguageCode(
    const std::string& chrome_locale) {
  for (size_t i = 0; i < arraysize(kLocaleToCLDLanguages); ++i) {
    if (chrome_locale == kLocaleToCLDLanguages[i].locale_language)
      return kLocaleToCLDLanguages[i].cld_language;
  }
  return chrome_locale;
}

// static
bool TranslationService::IsSupportedLanguage(const std::string& page_language) {
  if (supported_languages_.Pointer()->empty()) {
    for (size_t i = 0; i < arraysize(kSupportedLanguages); ++i)
      supported_languages_.Pointer()->insert(kSupportedLanguages[i]);
  }
  return supported_languages_.Pointer()->find(page_language) !=
      supported_languages_.Pointer()->end();
}

////////////////////////////////////////////////////////////////////////////////
// TranslationService, protected:

int TranslationService::GetSendRequestDelay() const {
  return kSendRequestDelay;
}

////////////////////////////////////////////////////////////////////////////////
// TranslationService, private:

void TranslationService::SendRequestToTranslationServer(
    TranslationRequest* request) {
  DCHECK(!request->query.empty());
  GURL url(request->secure ? kSecureServiceURL : kServiceURL);
  URLFetcher* url_fetcher =
      URLFetcher::Create(request->routing_id /* used in tests */,
                         url, URLFetcher::POST, this);
  url_fetcher->set_upload_data("application/x-www-form-urlencoded",
                               request->query);
  url_fetcher->set_request_context(Profile::GetDefaultRequestContext());
  url_fetcher->Start();

  // renderer_request_infos_ will now own the RendererRequestInfoList.
  renderer_request_infos_[url_fetcher] =
      request->renderer_request_info.release();

  // Remove the request from the translation request map.
  TranslationRequestMap& translation_request_map =
      request->secure ? pending_secure_translation_requests_ :
                        pending_translation_requests_;
  TranslationRequestMap::iterator iter =
      translation_request_map.find(request->routing_id);
  DCHECK(iter != translation_request_map.end());
  translation_request_map.erase(iter);
  delete request;
}

void TranslationService::SendResponseToRenderer(
    const URLFetcher* const_url_fetcher, int error_code,
    const TextChunksList& text_chunks_list) {
  scoped_ptr<const URLFetcher> url_fetcher(const_url_fetcher);
  RendererRequestInfoMap::iterator iter =
      renderer_request_infos_.find(url_fetcher.get());
  DCHECK(iter != renderer_request_infos_.end());
  scoped_ptr<RendererRequestInfoList> request_info_list(iter->second);
  DCHECK(error_code != 0 ||
         request_info_list->size() == text_chunks_list.size());
  for (size_t i = 0; i < request_info_list->size(); ++i) {
    RendererRequestInfo& request_info = request_info_list->at(i);
    message_sender_->Send(
        new ViewMsg_TranslateTextReponse(request_info.routing_id,
                                         request_info.work_id,
                                         error_code,
                                         error_code ? TextChunks() :
                                                      text_chunks_list[i]));
  }
  renderer_request_infos_.erase(iter);
}

void TranslationService::TranslationFailed(const URLFetcher* url_fetcher) {
  SendResponseToRenderer(url_fetcher, 1, TranslationService::TextChunksList());
}

string16 TranslationService::MergeTextChunks(const TextChunks& text_chunks) {
  // If there is only 1 chunk, we don't need an anchor tag as there is no order
  // to preserve.
  if (text_chunks.size() == 1U)
    return text_chunks[0];

  string16 str;
  for (size_t i = 0; i < text_chunks.size(); ++i) {
    str.append(kCRAnchorTagStart);
    str.append(IntToString16(i));
    str.append(kQuoteGreaterThan);
    str.append(text_chunks[i]);
    str.append(kClosingAnchorTag);
  }
  return str;
}

bool TranslationService::FindOpenTagIndex(const string16& text,
                                          size_t start_index,
                                          size_t* tag_start_index,
                                          size_t* tag_end_index,
                                          int* id) {
  DCHECK(tag_start_index && tag_end_index && id);
  size_t text_length = text.length();
  if (start_index >= text_length)
    return false;

  *tag_start_index = text.find(kCRAnchorTagStart, start_index);
  if (*tag_start_index == std::string::npos)
    return false;

  size_t quote_index = *tag_start_index + kCRAnchorTagStart.length();
  size_t close_quote_index = text.find(kQuote, quote_index);
  if (close_quote_index == std::string::npos) {
    NOTREACHED();
    return false;  // Not a valid anchor tag.
  }

  string16 id_str = text.substr(quote_index, close_quote_index - quote_index);
  // Get the id.
  if (!StringToInt(id_str, id)) {
    NOTREACHED();
    return false;  // Not a valid id, give up.
  }

  *tag_end_index = text.find(kGreaterThan, close_quote_index);
  if (*tag_end_index == std::string::npos || *tag_end_index >= text_length)
    return false;
  return true;
}

void TranslationService::SplitIntoTextChunks(const string16& translated_text,
                                             TextChunks* text_chunks) {
  int id = -1;
  size_t tag_start_index = 0;
  size_t tag_end_index = 0;
  if (!FindOpenTagIndex(translated_text, 0, &tag_start_index, &tag_end_index,
                        &id)) {
    // No magic anchor tag, it was a single chunk.
    text_chunks->push_back(translated_text);
    return;
  }

  // The server might send us some HTML with duplicated and unbalanced tags.
  // We separate from one tag begining to the next, and merge tags with
  // duplicate IDs.
  std::set<int> parsed_tags;
  string16 chunk;
  while (tag_start_index != std::string::npos) {
    int next_id = -1;
    size_t previous_tag_end_index = tag_end_index;
    if (!FindOpenTagIndex(translated_text, tag_end_index,
                          &tag_start_index, &tag_end_index, &next_id)) {
      // Last tag. Just report as one chunk.
      chunk = translated_text.substr(previous_tag_end_index + 1);
      tag_start_index = std::string::npos;  // So we break on next iteration.
    } else {
      // Extract the text for this tag.
      DCHECK(tag_start_index > previous_tag_end_index);
      chunk =
          translated_text.substr(previous_tag_end_index + 1,
                                 tag_start_index - previous_tag_end_index - 1);
    }
    chunk = RemoveTag(chunk);
    // The translation server leaves some ampersand character in the
    // translation.
    chunk = UnescapeForHTML(chunk);
    if (parsed_tags.count(id) > 0) {
      // We have already seen this tag, add it to the previous text-chunk.
      text_chunks->back().append(chunk);
    } else {
      text_chunks->push_back(chunk);
      parsed_tags.insert(id);
    }
    id = next_id;
  }
}

string16 TranslationService::RemoveTag(const string16& text) {
  // Remove any anchor tags, knowing they could be extra/unbalanced tags.
  string16 result;
  size_t start_index = text.find(kAnchorTagStart);
  if (start_index == std::string::npos) {
    result = text;
  } else {
    bool first_iter = true;
    while (true) {
      size_t stop_index = text.find(kGreaterThan, start_index);
      size_t next_tag_index = text.find(kLessThan, start_index + 1);
      // Ignore unclosed <a tag. (Ignore subsequent closing tags, they'll be
      // removed in the next loop.)
      if (stop_index == std::string::npos ||
          (next_tag_index != std::string::npos &&
           stop_index > next_tag_index)) {
        result.append(text.substr(start_index));
        break;
      }
      if (start_index > 0 && first_iter)
        result = text.substr(0, start_index);
      start_index = text.find(kAnchorTagStart, start_index + 1);
      if (start_index == std::string::npos) {
        result += text.substr(stop_index + 1);
        break;
      }
      result += text.substr(stop_index + 1, start_index - stop_index - 1);
      first_iter = false;
    }
  }

  // Now remove </a> tags.
  ReplaceSubstringsAfterOffset(&result, 0, kClosingAnchorTag, EmptyString16());
  return result;
}

// static
void TranslationService::AddTextToRequestString(std::string* request,
                                                const std::string& text,
                                                const std::string& source_lang,
                                                const std::string& target_lang,
                                                bool secure) {
  if (request->empty()) {
    // First request, add required parameters.
    request->append(kVersionParam);
    request->append("=1.0&");
    request->append(kClientParam);
    request->append("=cr&");  // cr = Chrome.
    request->append(kFormatParam);
    request->append("=html&");
    request->append(kLangPairParam);
    request->append("=");
    request->append(source_lang);
    request->append("%7C");  // | URL encoded.
    request->append(target_lang);
    if (secure) {
      request->append("&");
      request->append(kSSLParam);
      request->append("=1");
    }
  }
  // IMPORTANT NOTE: if you make any change below, make sure to reflect them in
  //                 text_param_length_ in TranslationService constructor.
  request->append("&");
  request->append(kTextParam);
  request->append("=");
  request->append(text);
}