Domain Reliability: Prepare for large batch update

BUG=

Review URL: https://codereview.chromium.org/615393004

Cr-Commit-Position: refs/heads/master@{#298359}

3 files changed, 126 insertions, 20 deletions

diff --git a/components/domain_reliability.gypi b/components/domain_reliability.gypi
index c210241..915f252 100644
--- a/components/domain_reliability.gypi
+++ b/components/domain_reliability.gypi
@@ -58,6 +58,7 @@
               'domain_reliability/baked_in_configs/ad_doubleclick_net.json',
               'domain_reliability/baked_in_configs/apis_google_com.json',
               'domain_reliability/baked_in_configs/c_admob_com.json',
+              'domain_reliability/baked_in_configs/clients2_google_com.json',
               'domain_reliability/baked_in_configs/csi_gstatic_com.json',
               'domain_reliability/baked_in_configs/ddm_google_com.json',
               'domain_reliability/baked_in_configs/docs_google_com.json',
@@ -84,18 +85,18 @@
               'domain_reliability/baked_in_configs/star_doubleclick_net.json',
               'domain_reliability/baked_in_configs/star_g_doubleclick_net.json',
               'domain_reliability/baked_in_configs/star_ggpht_com.json',
+              'domain_reliability/baked_in_configs/star_googleadservices_com.json',
+              'domain_reliability/baked_in_configs/star_googleapis_com.json',
               'domain_reliability/baked_in_configs/star_google_cn.json',
-              'domain_reliability/baked_in_configs/star_google_co_uk.json',
-              'domain_reliability/baked_in_configs/star_google_com.json',
               'domain_reliability/baked_in_configs/star_google_com_au.json',
+              'domain_reliability/baked_in_configs/star_google_com.json',
+              'domain_reliability/baked_in_configs/star_google_co_uk.json',
               'domain_reliability/baked_in_configs/star_google_de.json',
               'domain_reliability/baked_in_configs/star_google_fr.json',
               'domain_reliability/baked_in_configs/star_google_it.json',
               'domain_reliability/baked_in_configs/star_google_jp.json',
               'domain_reliability/baked_in_configs/star_google_org.json',
               'domain_reliability/baked_in_configs/star_google_ru.json',
-              'domain_reliability/baked_in_configs/star_googleadservices_com.json',
-              'domain_reliability/baked_in_configs/star_googleapis_com.json',
               'domain_reliability/baked_in_configs/star_googlesyndication_com.json',
               'domain_reliability/baked_in_configs/star_googleusercontent_com.json',
               'domain_reliability/baked_in_configs/star_googlevideo_com.json',
@@ -108,8 +109,8 @@
               'domain_reliability/baked_in_configs/t2_gstatic_com.json',
               'domain_reliability/baked_in_configs/t3_gstatic_com.json',
               'domain_reliability/baked_in_configs/themes_googleusercontent_com.json',
-              'domain_reliability/baked_in_configs/www_google_com.json',
               'domain_reliability/baked_in_configs/www_googleadservices_com.json',
+              'domain_reliability/baked_in_configs/www_google_com.json',
               'domain_reliability/baked_in_configs/www_gstatic_com.json',
               'domain_reliability/baked_in_configs/www_youtube_com.json',
             ],
@@ -121,9 +122,12 @@
           'outputs': [
             '<(baked_in_configs_cc)'
           ],
+          # The actual list of JSON files will overflow the command line length
+          # limit on Windows, so pass the name of the .gypi file and
+          # bake_in_configs.py will read the filenames out of it manually.
           'action': ['python',
                      '<(bake_in_configs_script)',
-                     '<@(baked_in_configs)',
+                     'domain_reliability.gypi',
                      '<(baked_in_configs_cc)'],
           'process_outputs_as_sources': 1,
           'message': 'Baking in Domain Reliability configs',
diff --git a/components/domain_reliability/BUILD.gn b/components/domain_reliability/BUILD.gn
index c0b2b4e..b68ffda 100644
--- a/components/domain_reliability/BUILD.gn
+++ b/components/domain_reliability/BUILD.gn
@@ -7,19 +7,74 @@ action("bake_in_configs") {
   script = "bake_in_configs.py"
 
   inputs = [
+    "baked_in_configs/accounts_google_com.json",
+    "baked_in_configs/ad_doubleclick_net.json",
     "baked_in_configs/apis_google_com.json",
+    "baked_in_configs/c_admob_com.json",
+    "baked_in_configs/clients2_google_com.json",
+    "baked_in_configs/csi_gstatic_com.json",
     "baked_in_configs/ddm_google_com.json",
+    "baked_in_configs/docs_google_com.json",
     "baked_in_configs/drive_google_com.json",
+    "baked_in_configs/e_admob_com.json",
+    "baked_in_configs/fonts_googleapis_com.json",
+    "baked_in_configs/googleads4_g_doubleclick_net.json",
+    "baked_in_configs/googleads_g_doubleclick_net.json",
+    "baked_in_configs/gstatic_com.json",
+    "baked_in_configs/lh3_ggpht_com.json",
+    "baked_in_configs/lh4_ggpht_com.json",
+    "baked_in_configs/lh5_ggpht_com.json",
+    "baked_in_configs/lh6_ggpht_com.json",
     "baked_in_configs/mail_google_com.json",
+    "baked_in_configs/media_admob_com.json",
+    "baked_in_configs/pagead2_googlesyndication_com.json",
+    "baked_in_configs/partner_googleadservices_com.json",
+    "baked_in_configs/pubads_g_doubleclick_net.json",
+    "baked_in_configs/redirector_googlevideo_com.json",
+    "baked_in_configs/redirector_gvt1_com.json",
+    "baked_in_configs/s0_2mdn_net.json",
     "baked_in_configs/ssl_gstatic_com.json",
+    "baked_in_configs/star_admob_com.json",
+    "baked_in_configs/star_doubleclick_net.json",
+    "baked_in_configs/star_g_doubleclick_net.json",
+    "baked_in_configs/star_ggpht_com.json",
+    "baked_in_configs/star_google_cn.json",
+    "baked_in_configs/star_google_co_uk.json",
+    "baked_in_configs/star_google_com.json",
+    "baked_in_configs/star_google_com_au.json",
+    "baked_in_configs/star_google_de.json",
+    "baked_in_configs/star_google_fr.json",
+    "baked_in_configs/star_google_it.json",
+    "baked_in_configs/star_google_jp.json",
+    "baked_in_configs/star_google_org.json",
+    "baked_in_configs/star_google_ru.json",
+    "baked_in_configs/star_googleadservices_com.json",
+    "baked_in_configs/star_googleapis_com.json",
+    "baked_in_configs/star_googlesyndication_com.json",
+    "baked_in_configs/star_googleusercontent_com.json",
+    "baked_in_configs/star_googlevideo_com.json",
+    "baked_in_configs/star_gstatic_com.json",
+    "baked_in_configs/star_gvt1_com.json",
+    "baked_in_configs/star_youtube_com.json",
+    "baked_in_configs/star_ytimg_com.json",
+    "baked_in_configs/t0_gstatic_com.json",
+    "baked_in_configs/t1_gstatic_com.json",
+    "baked_in_configs/t2_gstatic_com.json",
+    "baked_in_configs/t3_gstatic_com.json",
+    "baked_in_configs/themes_googleusercontent_com.json",
     "baked_in_configs/www_google_com.json",
+    "baked_in_configs/www_googleadservices_com.json",
+    "baked_in_configs/www_gstatic_com.json",
     "baked_in_configs/www_youtube_com.json",
   ]
   outputs = [
     "$target_gen_dir/baked_in_configs.cc",
   ]
 
-  args = rebase_path(inputs, root_build_dir) +
+  # The actual list of JSON files will overflow the command line length limit
+  # on Windows, so pass the name of the .gypi file and bake_in_configs.py will
+  # read the filenames out of it manually.
+  args = [ rebase_path("../domain_reliability.gypi", root_build_dir) ] +
          rebase_path(outputs, root_build_dir)
 }
 
diff --git a/components/domain_reliability/bake_in_configs.py b/components/domain_reliability/bake_in_configs.py
index 56f7aae..d1eb244 100755
--- a/components/domain_reliability/bake_in_configs.py
+++ b/components/domain_reliability/bake_in_configs.py
@@ -9,6 +9,7 @@ encodes their contents as an array of C strings that gets compiled in to Chrome
 and loaded at runtime."""
 
 
+import ast
 import json
 import os
 import sys
@@ -17,13 +18,31 @@ import sys
 # A whitelist of domains that the script will accept when baking configs in to
 # Chrome, to ensure incorrect ones are not added accidentally. Subdomains of
 # whitelist entries are also allowed (e.g. maps.google.com, ssl.gstatic.com).
-DOMAIN_WHITELIST = ('2mdn.net', 'admob.com', 'doubleclick.net', 'ggpht.com',
-                    'google.cn', 'google.co.uk', 'google.com', 'google.com.au',
-                    'google.de', 'google.fr', 'google.it', 'google.jp',
-                    'google.org', 'google.ru', 'googleadservices.com',
-                    'googleapis.com', 'googlesyndication.com',
-                    'googleusercontent.com', 'googlevideo.com', 'gstatic.com',
-                    'gvt1.com', 'youtube.com', 'ytimg.com')
+DOMAIN_WHITELIST = (
+  'admob.com',
+  'doubleclick.net',
+  'ggpht.com',
+  'google.cn',
+  'google.co.uk',
+  'google.com',
+  'google.com.au',
+  'google.de',
+  'google.fr',
+  'google.it',
+  'google.jp',
+  'google.org',
+  'google.ru',
+  'googleadservices.com',
+  'googleapis.com',
+  'googlesyndication.com',
+  'googleusercontent.com',
+  'googlevideo.com',
+  'gstatic.com',
+  'gvt1.com',
+  's0.2mdn.net',
+  'youtube.com',
+  'ytimg.com'
+)
 
 
 CC_HEADER = """// Copyright (C) 2014 The Chromium Authors. All rights reserved.
@@ -52,6 +71,26 @@ CC_FOOTER = """  NULL
 """
 
 
+def get_child(node, children_key, child_key, child_value):
+  children = node[children_key]
+  for child in children:
+    if child[child_key] == child_value:
+      return child
+  raise KeyError("Couldn't find child with %s = %s in %s" %
+                 (child_key, child_value, children_key))
+
+
+def read_json_files_from_gypi(gypi_file):
+  with open(gypi_file, 'r') as f:
+    gypi_text = f.read()
+  gypi_ast = ast.literal_eval(gypi_text)
+  target = get_child(gypi_ast, 'targets', 'target_name', 'domain_reliability')
+  action = get_child(target, 'actions', 'action_name', 'bake_in_configs')
+  json_files = action['variables']['baked_in_configs']
+  gypi_path = os.path.dirname(gypi_file)
+  return [ os.path.join(gypi_path, f) for f in json_files ]
+
+
 def domain_is_whitelisted(domain):
   return any(domain == e or domain.endswith('.' + e)  for e in DOMAIN_WHITELIST)
 
@@ -77,20 +116,28 @@ def quote_and_wrap_text(text, width=79, prefix='  "', suffix='"'):
 
 
 def main():
-  if len(sys.argv) < 3:
-    print >> sys.stderr, ('Usage: %s <JSON files...> <output C++ file>' %
+  if len(sys.argv) != 3:
+    print >> sys.stderr, ('Usage: %s <input .gypi file> <output C++ file>' %
                           sys.argv[0])
     print >> sys.stderr, sys.modules[__name__].__doc__
     return 1
+  gypi_file = sys.argv[1]
+  cpp_file = sys.argv[2]
+
+  json_files = read_json_files_from_gypi(gypi_file)
 
   cpp_code = CC_HEADER
   found_invalid_config = False
-  for json_file in sys.argv[1:-1]:
+  for json_file in json_files:
     with open(json_file, 'r') as f:
       json_text = f.read()
-    config = json.loads(json_text)
+    try:
+      config = json.loads(json_text)
+    except ValueError:
+      print >> sys.stderr, "%s: error parsing JSON" % json_file
+      raise
     if 'monitored_domain' not in config:
-      print >> sys.stderr, ('%s: no monitored_domain found' % json_file)
+      print >> sys.stderr, '%s: no monitored_domain found' % json_file
       found_invalid_config = True
       continue
     domain = config['monitored_domain']
@@ -107,7 +154,7 @@ def main():
   if found_invalid_config:
     return 1
 
-  with open(sys.argv[-1], 'wb') as f:
+  with open(cpp_file, 'wb') as f:
     f.write(cpp_code)
 
   return 0