[Sync] Add support for GetUpdates Throttling

During GetUpdates, if some data types got throttled, client
should backoff those data types and keep going with other unthrottled data types.

BUG=416992

Review URL: https://codereview.chromium.org/955693004

Cr-Commit-Position: refs/heads/master@{#318356}

17 files changed, 173 insertions, 30 deletions

diff --git a/sync/engine/commit.cc b/sync/engine/commit.cc
index 0561e9b..1811923 100644
--- a/sync/engine/commit.cc
+++ b/sync/engine/commit.cc
@@ -110,7 +110,7 @@ SyncerError Commit::PostAndProcessResponse(
 
   TRACE_EVENT_BEGIN0("sync", "PostCommit");
   const SyncerError post_result = SyncerProtoUtil::PostClientToServerMessage(
-      &message_, &response_, session);
+      &message_, &response_, session, NULL);
   TRACE_EVENT_END0("sync", "PostCommit");
 
   // TODO(rlarocque): Use result that includes errors captured later?
diff --git a/sync/engine/get_updates_processor.cc b/sync/engine/get_updates_processor.cc
index 25fdde4..a33c70d 100644
--- a/sync/engine/get_updates_processor.cc
+++ b/sync/engine/get_updates_processor.cc
@@ -78,8 +78,9 @@ void PartitionUpdatesByType(const sync_pb::GetUpdatesResponse& gu_response,
 
     TypeSyncEntityMap::iterator it = updates_by_type->find(type);
     if (it == updates_by_type->end()) {
-      NOTREACHED() << "Received update for unexpected type "
-                   << ModelTypeToString(type);
+      DLOG(WARNING)
+          << "Received update for unexpected type or the type is throttled:"
+          << ModelTypeToString(type);
       continue;
     }
 
@@ -168,7 +169,7 @@ GetUpdatesProcessor::GetUpdatesProcessor(UpdateHandlerMap* update_handler_map,
 GetUpdatesProcessor::~GetUpdatesProcessor() {}
 
 SyncerError GetUpdatesProcessor::DownloadUpdates(
-    ModelTypeSet request_types,
+    ModelTypeSet* request_types,
     sessions::SyncSession* session,
     bool create_mobile_bookmarks_folder) {
   TRACE_EVENT0("sync", "DownloadUpdates");
@@ -177,7 +178,7 @@ SyncerError GetUpdatesProcessor::DownloadUpdates(
   InitDownloadUpdatesContext(session,
                              create_mobile_bookmarks_folder,
                              &message);
-  PrepareGetUpdates(request_types, &message);
+  PrepareGetUpdates(*request_types, &message);
 
   SyncerError result = ExecuteDownloadUpdates(request_types, session, &message);
   session->mutable_status_controller()->set_last_download_updates_result(
@@ -209,7 +210,7 @@ void GetUpdatesProcessor::PrepareGetUpdates(
 }
 
 SyncerError GetUpdatesProcessor::ExecuteDownloadUpdates(
-    ModelTypeSet request_types,
+    ModelTypeSet* request_types,
     sessions::SyncSession* session,
     sync_pb::ClientToServerMessage* msg) {
   sync_pb::ClientToServerResponse update_response;
@@ -224,15 +225,17 @@ SyncerError GetUpdatesProcessor::ExecuteDownloadUpdates(
   session->SendProtocolEvent(
       *(delegate_.GetNetworkRequestEvent(base::Time::Now(), *msg)));
 
+  ModelTypeSet partial_failure_data_types;
+
   SyncerError result = SyncerProtoUtil::PostClientToServerMessage(
-      msg,
-      &update_response,
-      session);
+      msg, &update_response, session, &partial_failure_data_types);
 
   DVLOG(2) << SyncerProtoUtil::ClientToServerResponseDebugString(
       update_response);
 
-  if (result != SYNCER_OK) {
+  if (result == SERVER_RETURN_PARTIAL_FAILURE) {
+    request_types->RemoveAll(partial_failure_data_types);
+  } else if (result != SYNCER_OK) {
     GetUpdatesResponseEvent response_event(
         base::Time::Now(), update_response, result);
     session->SendProtocolEvent(response_event);
@@ -259,9 +262,8 @@ SyncerError GetUpdatesProcessor::ExecuteDownloadUpdates(
         HandleGetEncryptionKeyResponse(update_response, dir));
   }
 
-  SyncerError process_result = ProcessResponse(update_response.get_updates(),
-                                              request_types,
-                                              status);
+  SyncerError process_result =
+      ProcessResponse(update_response.get_updates(), *request_types, status);
 
   GetUpdatesResponseEvent response_event(
       base::Time::Now(), update_response, process_result);
diff --git a/sync/engine/get_updates_processor.h b/sync/engine/get_updates_processor.h
index 6c96132..934c3f1 100644
--- a/sync/engine/get_updates_processor.h
+++ b/sync/engine/get_updates_processor.h
@@ -54,7 +54,7 @@ class SYNC_EXPORT_PRIVATE GetUpdatesProcessor {
   // download succeeded but there are still some updates left to fetch on the
   // server, or an appropriate error value in case of failure.
   SyncerError DownloadUpdates(
-      ModelTypeSet request_types,
+      ModelTypeSet* request_types,
       sessions::SyncSession* session,
       bool create_mobile_bookmarks_folder);
 
@@ -71,7 +71,7 @@ class SYNC_EXPORT_PRIVATE GetUpdatesProcessor {
 
   // Sends the specified message to the server and stores the response in a
   // member of the |session|'s StatusController.
-  SyncerError ExecuteDownloadUpdates(ModelTypeSet request_types,
+  SyncerError ExecuteDownloadUpdates(ModelTypeSet* request_types,
                                      sessions::SyncSession* session,
                                      sync_pb::ClientToServerMessage* msg);
 
diff --git a/sync/engine/syncer.cc b/sync/engine/syncer.cc
index ef759b8..ac46881 100644
--- a/sync/engine/syncer.cc
+++ b/sync/engine/syncer.cc
@@ -66,7 +66,7 @@ bool Syncer::NormalSyncShare(ModelTypeSet request_types,
         session->context()->model_type_registry()->update_handler_map(),
         normal_delegate);
     if (!DownloadAndApplyUpdates(
-            request_types,
+            &request_types,
             session,
             &get_updates_processor,
             kCreateMobileBookmarksFolder)) {
@@ -95,7 +95,7 @@ bool Syncer::ConfigureSyncShare(
       session->context()->model_type_registry()->update_handler_map(),
       configure_delegate);
   DownloadAndApplyUpdates(
-      request_types,
+      &request_types,
       session,
       &get_updates_processor,
       kCreateMobileBookmarksFolder);
@@ -111,7 +111,7 @@ bool Syncer::PollSyncShare(ModelTypeSet request_types,
       session->context()->model_type_registry()->update_handler_map(),
       poll_delegate);
   DownloadAndApplyUpdates(
-      request_types,
+      &request_types,
       session,
       &get_updates_processor,
       kCreateMobileBookmarksFolder);
@@ -119,7 +119,7 @@ bool Syncer::PollSyncShare(ModelTypeSet request_types,
 }
 
 bool Syncer::DownloadAndApplyUpdates(
-    ModelTypeSet request_types,
+    ModelTypeSet* request_types,
     SyncSession* session,
     GetUpdatesProcessor* get_updates_processor,
     bool create_mobile_bookmarks_folder) {
@@ -146,7 +146,7 @@ bool Syncer::DownloadAndApplyUpdates(
     // Apply upates to the other types.  May or may not involve cross-thread
     // traffic, depending on the underlying update handlers and the GU type's
     // delegate.
-    get_updates_processor->ApplyUpdates(request_types,
+    get_updates_processor->ApplyUpdates(*request_types,
                                         session->mutable_status_controller());
 
     session->context()->set_hierarchy_conflict_detected(
diff --git a/sync/engine/syncer.h b/sync/engine/syncer.h
index ca93b3608..ef74877 100644
--- a/sync/engine/syncer.h
+++ b/sync/engine/syncer.h
@@ -70,7 +70,7 @@ class SYNC_EXPORT_PRIVATE Syncer {
 
  private:
   bool DownloadAndApplyUpdates(
-      ModelTypeSet request_types,
+      ModelTypeSet* request_types,
       sessions::SyncSession* session,
       GetUpdatesProcessor* get_updates_processor,
       bool create_mobile_bookmarks_folder);
diff --git a/sync/engine/syncer_proto_util.cc b/sync/engine/syncer_proto_util.cc
index 44ac321..c28d7c2 100644
--- a/sync/engine/syncer_proto_util.cc
+++ b/sync/engine/syncer_proto_util.cc
@@ -124,6 +124,8 @@ SyncProtocolErrorType ConvertSyncProtocolErrorTypePBToLocalType(
       return DISABLED_BY_ADMIN;
     case sync_pb::SyncEnums::USER_ROLLBACK:
       return USER_ROLLBACK;
+    case sync_pb::SyncEnums::PARTIAL_FAILURE:
+      return PARTIAL_FAILURE;
     case sync_pb::SyncEnums::UNKNOWN:
       return UNKNOWN_ERROR;
     case sync_pb::SyncEnums::USER_NOT_ACTIVATED:
@@ -184,8 +186,9 @@ SyncProtocolError ConvertErrorPBToLocalType(
       error.action());
 
   if (error.error_data_type_ids_size() > 0) {
-    // THROTTLED is currently the only error code that uses |error_data_types|.
-    DCHECK_EQ(error.error_type(), sync_pb::SyncEnums::THROTTLED);
+    // THROTTLED and PARTIAL_FAILURE are currently the only error codes
+    // that uses |error_data_types|.
+    // In both cases, |error_data_types| are throttled.
     for (int i = 0; i < error.error_data_type_ids_size(); ++i) {
       int field_number = error.error_data_type_ids(i);
       ModelType model_type =
@@ -345,7 +348,8 @@ SyncProtocolError ConvertLegacyErrorCodeToNewError(
 SyncerError SyncerProtoUtil::PostClientToServerMessage(
     ClientToServerMessage* msg,
     ClientToServerResponse* response,
-    SyncSession* session) {
+    SyncSession* session,
+    ModelTypeSet* partial_failure_data_types) {
   CHECK(response);
   DCHECK(!msg->get_updates().has_from_timestamp());  // Deprecated.
   DCHECK(!msg->get_updates().has_requested_types());  // Deprecated.
@@ -490,6 +494,17 @@ SyncerError SyncerProtoUtil::PostClientToServerMessage(
       return SERVER_RETURN_DISABLED_BY_ADMIN;
     case USER_ROLLBACK:
       return SERVER_RETURN_USER_ROLLBACK;
+    case PARTIAL_FAILURE:
+      // This only happens when partial throttling during GetUpdates.
+      if (!sync_protocol_error.error_data_types.Empty()) {
+        DLOG(WARNING) << "Some types throttled by syncer during GetUpdates.";
+        session->delegate()->OnTypesThrottled(
+            sync_protocol_error.error_data_types, GetThrottleDelay(*response));
+      }
+      if (partial_failure_data_types != NULL) {
+        *partial_failure_data_types = sync_protocol_error.error_data_types;
+      }
+      return SERVER_RETURN_PARTIAL_FAILURE;
     default:
       NOTREACHED();
       return UNSET;
diff --git a/sync/engine/syncer_proto_util.h b/sync/engine/syncer_proto_util.h
index 3fab8b5..604fbc6 100644
--- a/sync/engine/syncer_proto_util.h
+++ b/sync/engine/syncer_proto_util.h
@@ -56,7 +56,8 @@ class SYNC_EXPORT_PRIVATE SyncerProtoUtil {
   static SyncerError PostClientToServerMessage(
       sync_pb::ClientToServerMessage* msg,
       sync_pb::ClientToServerResponse* response,
-      sessions::SyncSession* session);
+      sessions::SyncSession* session,
+      ModelTypeSet* partial_failure_data_types);
 
   static bool ShouldMaintainPosition(const sync_pb::SyncEntity& sync_entity);
 
diff --git a/sync/engine/syncer_unittest.cc b/sync/engine/syncer_unittest.cc
index 5f049ec..fcc45d3c 100644
--- a/sync/engine/syncer_unittest.cc
+++ b/sync/engine/syncer_unittest.cc
@@ -23,6 +23,7 @@
 #include "base/strings/stringprintf.h"
 #include "base/time/time.h"
 #include "build/build_config.h"
+#include "sync/engine/backoff_delay_provider.h"
 #include "sync/engine/get_commit_ids.h"
 #include "sync/engine/net/server_connection_manager.h"
 #include "sync/engine/sync_scheduler_impl.h"
@@ -195,7 +196,7 @@ class SyncerTest : public testing::Test,
   }
   void OnTypesThrottled(ModelTypeSet types,
                         const base::TimeDelta& throttle_duration) override {
-    FAIL() << "Should not get silenced.";
+    scheduler_->OnTypesThrottled(types, throttle_duration);
   }
   bool IsCurrentlyThrottled() override { return false; }
   void OnReceivedLongPollIntervalUpdate(
@@ -308,6 +309,12 @@ class SyncerTest : public testing::Test,
         "fake_invalidator_client_id"));
     context_->SetRoutingInfo(routing_info);
     syncer_ = new Syncer(&cancelation_signal_);
+    scheduler_.reset(new SyncSchedulerImpl(
+        "TestSyncScheduler",
+        BackoffDelayProvider::FromDefaults(),
+        context_.get(),
+        // scheduler_ owned syncer_ now and will manage the memory of syncer_
+        syncer_));
 
     syncable::ReadTransaction trans(FROM_HERE, directory());
     syncable::Directory::Metahandles children;
@@ -325,8 +332,7 @@ class SyncerTest : public testing::Test,
     model_type_registry_->UnregisterDirectoryTypeDebugInfoObserver(
         &debug_info_cache_);
     mock_server_.reset();
-    delete syncer_;
-    syncer_ = NULL;
+    scheduler_.reset();
     dir_maker_.TearDown();
   }
 
@@ -585,6 +591,7 @@ class SyncerTest : public testing::Test,
   TypeDebugInfoCache debug_info_cache_;
   MockNudgeHandler mock_nudge_handler_;
   scoped_ptr<ModelTypeRegistry> model_type_registry_;
+  scoped_ptr<SyncSchedulerImpl> scheduler_;
   scoped_ptr<SyncSessionContext> context_;
   bool saw_syncer_event_;
   base::TimeDelta last_short_poll_interval_received_;
@@ -794,6 +801,87 @@ TEST_F(SyncerTest, GetCommitIdsFiltersUnreadyEntries) {
   }
 }
 
+TEST_F(SyncerTest, GetUpdatesPartialThrottled) {
+  sync_pb::EntitySpecifics bookmark, pref;
+  bookmark.mutable_bookmark()->set_title("title");
+  pref.mutable_preference()->set_name("name");
+  AddDefaultFieldValue(BOOKMARKS, &bookmark);
+  AddDefaultFieldValue(PREFERENCES, &pref);
+
+  // Normal sync, all the data types should get synced.
+  mock_server_->AddUpdateSpecifics(1, 0, "A", 10, 10, true, 0, bookmark,
+                                   foreign_cache_guid(), "-1");
+  mock_server_->AddUpdateSpecifics(2, 1, "B", 10, 10, false, 2, bookmark,
+                                   foreign_cache_guid(), "-2");
+  mock_server_->AddUpdateSpecifics(3, 1, "C", 10, 10, false, 1, bookmark,
+                                   foreign_cache_guid(), "-3");
+  mock_server_->AddUpdateSpecifics(4, 0, "D", 10, 10, false, 0, pref);
+
+  SyncShareNudge();
+  {
+    // Initial state. Everything is normal.
+    syncable::ReadTransaction rtrans(FROM_HERE, directory());
+    VERIFY_ENTRY(1, false, false, false, 0, 10, 10, ids_, &rtrans);
+    VERIFY_ENTRY(2, false, false, false, 1, 10, 10, ids_, &rtrans);
+    VERIFY_ENTRY(3, false, false, false, 1, 10, 10, ids_, &rtrans);
+    VERIFY_ENTRY(4, false, false, false, 0, 10, 10, ids_, &rtrans);
+  }
+
+  // Set BOOKMARKS throttled but PREFERENCES not,
+  // then BOOKMARKS should not get synced but PREFERENCES should.
+  ModelTypeSet throttled_types(BOOKMARKS);
+  mock_server_->set_partial_throttling(true);
+  mock_server_->SetThrottledTypes(throttled_types);
+
+  mock_server_->AddUpdateSpecifics(1, 0, "E", 20, 20, true, 0, bookmark,
+                                   foreign_cache_guid(), "-1");
+  mock_server_->AddUpdateSpecifics(2, 1, "F", 20, 20, false, 2, bookmark,
+                                   foreign_cache_guid(), "-2");
+  mock_server_->AddUpdateSpecifics(3, 1, "G", 20, 20, false, 1, bookmark,
+                                   foreign_cache_guid(), "-3");
+  mock_server_->AddUpdateSpecifics(4, 0, "H", 20, 20, false, 0, pref);
+  {
+    WriteTransaction wtrans(FROM_HERE, UNITTEST, directory());
+    MutableEntry A(&wtrans, GET_BY_ID, ids_.FromNumber(1));
+    MutableEntry B(&wtrans, GET_BY_ID, ids_.FromNumber(2));
+    MutableEntry C(&wtrans, GET_BY_ID, ids_.FromNumber(3));
+    MutableEntry D(&wtrans, GET_BY_ID, ids_.FromNumber(4));
+    A.PutIsUnsynced(true);
+    B.PutIsUnsynced(true);
+    C.PutIsUnsynced(true);
+    D.PutIsUnsynced(true);
+  }
+  SyncShareNudge();
+  {
+    // BOOKMARKS throttled.
+    syncable::ReadTransaction rtrans(FROM_HERE, directory());
+    VERIFY_ENTRY(1, false, true, false, 0, 10, 10, ids_, &rtrans);
+    VERIFY_ENTRY(2, false, true, false, 1, 10, 10, ids_, &rtrans);
+    VERIFY_ENTRY(3, false, true, false, 1, 10, 10, ids_, &rtrans);
+    VERIFY_ENTRY(4, false, false, false, 0, 21, 21, ids_, &rtrans);
+  }
+
+  // Unthrottled BOOKMARKS, then BOOKMARKS should get synced now.
+  mock_server_->set_partial_throttling(false);
+
+  mock_server_->AddUpdateSpecifics(1, 0, "E", 30, 30, true, 0, bookmark,
+                                   foreign_cache_guid(), "-1");
+  mock_server_->AddUpdateSpecifics(2, 1, "F", 30, 30, false, 2, bookmark,
+                                   foreign_cache_guid(), "-2");
+  mock_server_->AddUpdateSpecifics(3, 1, "G", 30, 30, false, 1, bookmark,
+                                   foreign_cache_guid(), "-3");
+  mock_server_->AddUpdateSpecifics(4, 0, "H", 30, 30, false, 0, pref);
+  SyncShareNudge();
+  {
+    // BOOKMARKS unthrottled.
+    syncable::ReadTransaction rtrans(FROM_HERE, directory());
+    VERIFY_ENTRY(1, false, false, false, 0, 31, 31, ids_, &rtrans);
+    VERIFY_ENTRY(2, false, false, false, 1, 31, 31, ids_, &rtrans);
+    VERIFY_ENTRY(3, false, false, false, 1, 31, 31, ids_, &rtrans);
+    VERIFY_ENTRY(4, false, false, false, 0, 30, 30, ids_, &rtrans);
+  }
+}
+
 // This test uses internal knowledge of the directory to test correctness of
 // GetCommitIds.  In almost every other test, the hierarchy is created from
 // parent to child order, and so parents always have metahandles that are
diff --git a/sync/internal_api/public/util/syncer_error.cc b/sync/internal_api/public/util/syncer_error.cc
index 8130b1e..5c59e92 100644
--- a/sync/internal_api/public/util/syncer_error.cc
+++ b/sync/internal_api/public/util/syncer_error.cc
@@ -28,6 +28,7 @@ const char* GetSyncerErrorString(SyncerError value) {
     ENUM_CASE(SERVER_RESPONSE_VALIDATION_FAILED);
     ENUM_CASE(SERVER_RETURN_DISABLED_BY_ADMIN);
     ENUM_CASE(SERVER_RETURN_USER_ROLLBACK);
+    ENUM_CASE(SERVER_RETURN_PARTIAL_FAILURE);
     ENUM_CASE(SERVER_MORE_TO_DOWNLOAD);
     ENUM_CASE(DATATYPE_TRIGGERED_RETRY);
     ENUM_CASE(SYNCER_OK);
diff --git a/sync/internal_api/public/util/syncer_error.h b/sync/internal_api/public/util/syncer_error.h
index 05ad963..9a9219f 100644
--- a/sync/internal_api/public/util/syncer_error.h
+++ b/sync/internal_api/public/util/syncer_error.h
@@ -31,6 +31,7 @@ enum SYNC_EXPORT_PRIVATE SyncerError {
   SERVER_RESPONSE_VALIDATION_FAILED,
   SERVER_RETURN_DISABLED_BY_ADMIN,
   SERVER_RETURN_USER_ROLLBACK,
+  SERVER_RETURN_PARTIAL_FAILURE,
 
   // A datatype decided the sync cycle needed to be performed again.
   DATATYPE_TRIGGERED_RETRY,
diff --git a/sync/protocol/proto_enum_conversions.cc b/sync/protocol/proto_enum_conversions.cc
index 7be7453..9b7e7a4 100644
--- a/sync/protocol/proto_enum_conversions.cc
+++ b/sync/protocol/proto_enum_conversions.cc
@@ -166,6 +166,7 @@ const char* GetErrorTypeString(sync_pb::SyncEnums::ErrorType error_type) {
     ENUM_CASE(sync_pb::SyncEnums, MIGRATION_DONE);
     ENUM_CASE(sync_pb::SyncEnums, DISABLED_BY_ADMIN);
     ENUM_CASE(sync_pb::SyncEnums, USER_ROLLBACK);
+    ENUM_CASE(sync_pb::SyncEnums, PARTIAL_FAILURE);
     ENUM_CASE(sync_pb::SyncEnums, UNKNOWN);
   }
   NOTREACHED();
diff --git a/sync/protocol/sync.proto b/sync/protocol/sync.proto
index 78f9d40..d5a75cb 100644
--- a/sync/protocol/sync.proto
+++ b/sync/protocol/sync.proto
@@ -966,8 +966,11 @@ message ClientToServerResponse {
     optional string url                     = 3;
     optional SyncEnums.Action action        = 4 [default = UNKNOWN_ACTION];
 
-    // Currently only meaningful if |error_type| is throttled. If this field
-    // is absent then the whole client (all datatypes) is throttled.
+    // Currently meaningful if |error_type| is throttled or partial_failure.
+    // In the throttled case, if this field is absent then the whole client
+    // (all datatypes) is throttled.
+    // In the partial_failure case, this field denotes partial failures. The
+    // client should retry those datatypes with exponential backoff.
     repeated int32 error_data_type_ids = 5;
   }
   optional Error error = 13;
diff --git a/sync/protocol/sync_enums.proto b/sync/protocol/sync_enums.proto
index ec1ee65..35614ba 100644
--- a/sync/protocol/sync_enums.proto
+++ b/sync/protocol/sync_enums.proto
@@ -98,6 +98,8 @@ message SyncEnums {
                               // a server.
     DISABLED_BY_ADMIN = 10;   // An administrator disabled sync for this domain.
     USER_ROLLBACK     = 11;   // Client told to stop syncing and roll back.
+    PARTIAL_FAILURE   = 12;   // Return when client want to update several data
+                              // types, but some of them failed(e.g. throttled).
     UNKNOWN            = 100; // Unknown value. This should never be explicitly
                               // used; it is the default value when an
                               // out-of-date client parses a value it doesn't
diff --git a/sync/protocol/sync_protocol_error.cc b/sync/protocol/sync_protocol_error.cc
index 713137d..e5a7393 100644
--- a/sync/protocol/sync_protocol_error.cc
+++ b/sync/protocol/sync_protocol_error.cc
@@ -24,6 +24,7 @@ const char* GetSyncErrorTypeString(SyncProtocolErrorType type) {
     ENUM_CASE(INVALID_CREDENTIAL);
     ENUM_CASE(DISABLED_BY_ADMIN);
     ENUM_CASE(USER_ROLLBACK);
+    ENUM_CASE(PARTIAL_FAILURE);
     ENUM_CASE(UNKNOWN_ERROR);
   }
   NOTREACHED();
diff --git a/sync/protocol/sync_protocol_error.h b/sync/protocol/sync_protocol_error.h
index 0c825b7..6941fc8 100644
--- a/sync/protocol/sync_protocol_error.h
+++ b/sync/protocol/sync_protocol_error.h
@@ -45,6 +45,9 @@ enum SyncProtocolErrorType {
   // Client told to stop syncing this device and roll back local data.
   USER_ROLLBACK,
 
+  // Some of servers are busy. Try later with busy servers.
+  PARTIAL_FAILURE,
+
   // The default value.
   UNKNOWN_ERROR
 };
diff --git a/sync/test/engine/mock_connection_manager.cc b/sync/test/engine/mock_connection_manager.cc
index 4e42f54..8a4ba9c 100644
--- a/sync/test/engine/mock_connection_manager.cc
+++ b/sync/test/engine/mock_connection_manager.cc
@@ -47,6 +47,7 @@ MockConnectionManager::MockConnectionManager(syncable::Directory* directory,
       directory_(directory),
       mid_commit_observer_(NULL),
       throttling_(false),
+      partialThrottling_(false),
       fail_with_auth_invalid_(false),
       fail_non_periodic_get_updates_(false),
       next_position_in_parent_(2),
@@ -154,6 +155,18 @@ bool MockConnectionManager::PostBufferToPath(PostBufferParams* params,
       throttling_ = false;
     }
 
+    if (partialThrottling_) {
+      sync_pb::ClientToServerResponse_Error* response_error =
+          response.mutable_error();
+      response_error->set_error_type(SyncEnums::PARTIAL_FAILURE);
+      for (ModelTypeSet::Iterator it = throttled_type_.First(); it.Good();
+           it.Inc()) {
+        response_error->add_error_data_type_ids(
+            GetSpecificsFieldNumberFromModelType(it.Get()));
+      }
+      partialThrottling_ = false;
+    }
+
     if (fail_with_auth_invalid_)
       response.set_error_code(SyncEnums::AUTH_INVALID);
   }
diff --git a/sync/test/engine/mock_connection_manager.h b/sync/test/engine/mock_connection_manager.h
index 611d949..8a1bbea 100644
--- a/sync/test/engine/mock_connection_manager.h
+++ b/sync/test/engine/mock_connection_manager.h
@@ -219,6 +219,8 @@ class MockConnectionManager : public ServerConnectionManager {
     store_birthday_ = new_birthday;
   }
 
+  void set_partial_throttling(bool value) { partialThrottling_ = value; }
+
   // Retrieve the number of GetUpdates requests that the mock server has
   // seen since the last time this function was called.  Can be used to
   // verify that a GetUpdates actually did or did not happen after running
@@ -235,6 +237,9 @@ class MockConnectionManager : public ServerConnectionManager {
     expected_filter_ = expected_filter;
   }
 
+  // Set throttled date types.
+  void SetThrottledTypes(ModelTypeSet types) { throttled_type_ = types; }
+
   void SetServerReachable();
 
   void SetServerNotReachable();
@@ -368,6 +373,11 @@ class MockConnectionManager : public ServerConnectionManager {
   // Protected by |response_code_override_lock_|.
   bool throttling_;
 
+  // Whether we are faking a server mandating clients to partial throttle
+  // requests.
+  // Protected by |response_code_override_lock_|.
+  bool partialThrottling_;
+
   // Whether we are failing all requests by returning
   // ClientToServerResponse::AUTH_INVALID.
   // Protected by |response_code_override_lock_|.
@@ -391,6 +401,8 @@ class MockConnectionManager : public ServerConnectionManager {
 
   ModelTypeSet expected_filter_;
 
+  ModelTypeSet throttled_type_;
+
   int num_get_updates_requests_;
 
   std::string next_token_;