summaryrefslogtreecommitdiffstats
path: root/build/get_syzygy_binaries.py
blob: 05e8072d3dfa48458b7076b53b7f8a85fc3effc9 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A utility script for downloading versioned Syzygy binaries."""

import cStringIO
import hashlib
import errno
import json
import logging
import optparse
import os
import re
import shutil
import stat
import sys
import subprocess
import urllib2
import zipfile


_LOGGER = logging.getLogger(os.path.basename(__file__))

# The URL where official builds are archived.
_SYZYGY_ARCHIVE_URL = ('http://syzygy-archive.commondatastorage.googleapis.com/'
    'builds/official/%(revision)s')

# A JSON file containing the state of the download directory. If this file and
# directory state do not agree, then the binaries will be downloaded and
# installed again.
_STATE = '.state'

# This matches an integer (an SVN revision number) or a SHA1 value (a GIT hash).
# The archive exclusively uses lowercase GIT hashes.
_REVISION_RE = re.compile('^(?:\d+|[a-f0-9]{40})$')

# This matches an MD5 hash.
_MD5_RE = re.compile('^[a-f0-9]{32}$')

# List of reources to be downloaded and installed. These are tuples with the
# following format:
# (basename, logging name, relative installation path, extraction filter)
_RESOURCES = [
  ('benchmark.zip', 'benchmark', '', None),
  ('binaries.zip', 'binaries', 'exe', None),
  ('symbols.zip', 'symbols', 'exe',
      lambda x: x.filename.endswith('.dll.pdb')),
  ('include.zip', 'include', 'include', None),
  ('lib.zip', 'library', 'lib', None)]


def _Shell(*cmd, **kw):
  """Runs |cmd|, returns the results from Popen(cmd).communicate()."""
  _LOGGER.debug('Executing %s.', cmd)
  prog = subprocess.Popen(cmd, shell=True, **kw)

  stdout, stderr = prog.communicate()
  if prog.returncode != 0:
    raise RuntimeError('Command "%s" returned %d.' % (cmd, prog.returncode))
  return (stdout, stderr)


def _LoadState(output_dir):
  """Loads the contents of the state file for a given |output_dir|, returning
  None if it doesn't exist.
  """
  path = os.path.join(output_dir, _STATE)
  if not os.path.exists(path):
    _LOGGER.debug('No state file found.')
    return None
  with open(path, 'rb') as f:
    _LOGGER.debug('Reading state file: %s', path)
    try:
      return json.load(f)
    except ValueError:
      _LOGGER.debug('Invalid state file.')
      return None


def _SaveState(output_dir, state, dry_run=False):
  """Saves the |state| dictionary to the given |output_dir| as a JSON file."""
  path = os.path.join(output_dir, _STATE)
  _LOGGER.debug('Writing state file: %s', path)
  if dry_run:
    return
  with open(path, 'wb') as f:
    f.write(json.dumps(state, sort_keys=True, indent=2))


def _Md5(path):
  """Returns the MD5 hash of the file at |path|, which must exist."""
  return hashlib.md5(open(path, 'rb').read()).hexdigest()


def _StateIsValid(state):
  """Returns true if the given state structure is valid."""
  if not isinstance(state, dict):
    _LOGGER.debug('State must be a dict.')
    return False
  r = state.get('revision', None)
  if not isinstance(r, basestring) or not _REVISION_RE.match(r):
    _LOGGER.debug('State contains an invalid revision.')
    return False
  c = state.get('contents', None)
  if not isinstance(c, dict):
    _LOGGER.debug('State must contain a contents dict.')
    return False
  for (relpath, md5) in c.iteritems():
    if not isinstance(relpath, basestring) or len(relpath) == 0:
      _LOGGER.debug('State contents dict contains an invalid path.')
      return False
    if not isinstance(md5, basestring) or not _MD5_RE.match(md5):
      _LOGGER.debug('State contents dict contains an invalid MD5 digest.')
      return False
  return True


def _BuildActualState(stored, revision, output_dir):
  """Builds the actual state using the provided |stored| state as a template.
  Only examines files listed in the stored state, causing the script to ignore
  files that have been added to the directories locally. |stored| must be a
  valid state dictionary.
  """
  contents = {}
  state = { 'revision': revision, 'contents': contents }
  for relpath, md5 in stored['contents'].iteritems():
    abspath = os.path.abspath(os.path.join(output_dir, relpath))
    if os.path.isfile(abspath):
      m = _Md5(abspath)
      contents[relpath] = m

  return state


def _StatesAreConsistent(stored, actual):
  """Validates whether two state dictionaries are consistent. Both must be valid
  state dictionaries. Additional entries in |actual| are ignored.
  """
  if stored['revision'] != actual['revision']:
    _LOGGER.debug('Mismatched revision number.')
    return False
  cont_stored = stored['contents']
  cont_actual = actual['contents']
  for relpath, md5 in cont_stored.iteritems():
    if relpath not in cont_actual:
      _LOGGER.debug('Missing content: %s', relpath)
      return False
    if md5 != cont_actual[relpath]:
      _LOGGER.debug('Modified content: %s', relpath)
      return False
  return True


def _GetCurrentState(revision, output_dir):
  """Loads the current state and checks to see if it is consistent. Returns
  a tuple (state, bool). The returned state will always be valid, even if an
  invalid state is present on disk.
  """
  stored = _LoadState(output_dir)
  if not _StateIsValid(stored):
    _LOGGER.debug('State is invalid.')
    # Return a valid but empty state.
    return ({'revision': '0', 'contents': {}}, False)
  actual = _BuildActualState(stored, revision, output_dir)
  # If the script has been modified consider the state invalid.
  path = os.path.join(output_dir, _STATE)
  if os.path.getmtime(__file__) > os.path.getmtime(path):
    return (stored, False)
  # Otherwise, explicitly validate the state.
  if not _StatesAreConsistent(stored, actual):
    return (stored, False)
  return (stored, True)


def _DirIsEmpty(path):
  """Returns true if the given directory is empty, false otherwise."""
  for root, dirs, files in os.walk(path):
    return not dirs and not files


def _RmTreeHandleReadOnly(func, path, exc):
  """An error handling function for use with shutil.rmtree. This will
  detect failures to remove read-only files, and will change their properties
  prior to removing them. This is necessary on Windows as os.remove will return
  an access error for read-only files, and git repos contain read-only
  pack/index files.
  """
  excvalue = exc[1]
  if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
    _LOGGER.debug('Removing read-only path: %s', path)
    os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
    func(path)
  else:
    raise


def _RmTree(path):
  """A wrapper of shutil.rmtree that handles read-only files."""
  shutil.rmtree(path, ignore_errors=False, onerror=_RmTreeHandleReadOnly)


def _CleanState(output_dir, state, dry_run=False):
  """Cleans up files/directories in |output_dir| that are referenced by
  the given |state|. Raises an error if there are local changes. Returns a
  dictionary of files that were deleted.
  """
  _LOGGER.debug('Deleting files from previous installation.')
  deleted = {}

  # Generate a list of files to delete, relative to |output_dir|.
  contents = state['contents']
  files = sorted(contents.keys())

  # Try to delete the files. Keep track of directories to delete as well.
  dirs = {}
  for relpath in files:
    fullpath = os.path.join(output_dir, relpath)
    fulldir = os.path.dirname(fullpath)
    dirs[fulldir] = True
    if os.path.exists(fullpath):
      # If somehow the file has become a directory complain about it.
      if os.path.isdir(fullpath):
        raise Exception('Directory exists where file expected: %s' % fullpath)

      # Double check that the file doesn't have local changes. If it does
      # then refuse to delete it.
      if relpath in contents:
        stored_md5 = contents[relpath]
        actual_md5 = _Md5(fullpath)
        if actual_md5 != stored_md5:
          raise Exception('File has local changes: %s' % fullpath)

      # The file is unchanged so it can safely be deleted.
      _LOGGER.debug('Deleting file "%s".', fullpath)
      deleted[relpath] = True
      if not dry_run:
        os.unlink(fullpath)

  # Sort directories from longest name to shortest. This lets us remove empty
  # directories from the most nested paths first.
  dirs = sorted(dirs.keys(), key=lambda x: len(x), reverse=True)
  for p in dirs:
    if os.path.exists(p) and _DirIsEmpty(p):
      _LOGGER.debug('Deleting empty directory "%s".', p)
      if not dry_run:
        _RmTree(p)

  return deleted


def _Download(url):
  """Downloads the given URL and returns the contents as a string."""
  response = urllib2.urlopen(url)
  if response.code != 200:
    raise RuntimeError('Failed to download "%s".' % url)
  return response.read()


def _InstallBinaries(options, deleted={}):
  """Installs Syzygy binaries. This assumes that the output directory has
  already been cleaned, as it will refuse to overwrite existing files."""
  contents = {}
  state = { 'revision': options.revision, 'contents': contents }
  archive_url = _SYZYGY_ARCHIVE_URL % { 'revision': options.revision }
  for (base, name, subdir, filt) in _RESOURCES:
    # Create the output directory if it doesn't exist.
    fulldir = os.path.join(options.output_dir, subdir)
    if os.path.isfile(fulldir):
      raise Exception('File exists where a directory needs to be created: %s' %
                      fulldir)
    if not os.path.exists(fulldir):
      _LOGGER.debug('Creating directory: %s', fulldir)
      if not options.dry_run:
        os.makedirs(fulldir)

    # Download the archive.
    url = archive_url + '/' + base
    _LOGGER.debug('Retrieving %s archive at "%s".', name, url)
    data = _Download(url)

    _LOGGER.debug('Unzipping %s archive.', name)
    archive = zipfile.ZipFile(cStringIO.StringIO(data))
    for entry in archive.infolist():
      if not filt or filt(entry):
        fullpath = os.path.normpath(os.path.join(fulldir, entry.filename))
        relpath = os.path.relpath(fullpath, options.output_dir)
        if os.path.exists(fullpath):
          # If in a dry-run take into account the fact that the file *would*
          # have been deleted.
          if options.dry_run and relpath in deleted:
            pass
          else:
            raise Exception('Path already exists: %s' % fullpath)

        # Extract the file and update the state dictionary.
        _LOGGER.debug('Extracting "%s".', fullpath)
        if not options.dry_run:
          archive.extract(entry.filename, fulldir)
          md5 = _Md5(fullpath)
          contents[relpath] = md5
          if sys.platform == 'cygwin':
            os.chmod(fullpath, os.stat(fullpath).st_mode | stat.S_IXUSR)

  return state


def _ParseCommandLine():
  """Parses the command-line and returns an options structure."""
  option_parser = optparse.OptionParser()
  option_parser.add_option('--dry-run', action='store_true', default=False,
      help='If true then will simply list actions that would be performed.')
  option_parser.add_option('--force', action='store_true', default=False,
      help='Force an installation even if the binaries are up to date.')
  option_parser.add_option('--output-dir', type='string',
      help='The path where the binaries will be replaced. Existing binaries '
           'will only be overwritten if not up to date.')
  option_parser.add_option('--overwrite', action='store_true', default=False,
      help='If specified then the installation will happily delete and rewrite '
           'the entire output directory, blasting any local changes.')
  option_parser.add_option('--revision', type='string',
      help='The SVN revision or GIT hash associated with the required version.')
  option_parser.add_option('--revision-file', type='string',
      help='A text file containing an SVN revision or GIT hash.')
  option_parser.add_option('--verbose', dest='log_level', action='store_const',
      default=logging.INFO, const=logging.DEBUG,
      help='Enables verbose logging.')
  option_parser.add_option('--quiet', dest='log_level', action='store_const',
      default=logging.INFO, const=logging.ERROR,
      help='Disables all output except for errors.')
  options, args = option_parser.parse_args()
  if args:
    option_parser.error('Unexpected arguments: %s' % args)
  if not options.output_dir:
    option_parser.error('Must specify --output-dir.')
  if not options.revision and not options.revision_file:
    option_parser.error('Must specify one of --revision or --revision-file.')
  if options.revision and options.revision_file:
    option_parser.error('Must not specify both --revision and --revision-file.')

  # Configure logging.
  logging.basicConfig(level=options.log_level)

  # If a revision file has been specified then read it.
  if options.revision_file:
    options.revision = open(options.revision_file, 'rb').read().strip()
    _LOGGER.debug('Parsed revision "%s" from file "%s".',
                 options.revision, options.revision_file)

  # Ensure that the specified SVN revision or GIT hash is valid.
  if not _REVISION_RE.match(options.revision):
    option_parser.error('Must specify a valid SVN or GIT revision.')

  # This just makes output prettier to read.
  options.output_dir = os.path.normpath(options.output_dir)

  return options


def main():
  # We only care about Windows platforms, as the Syzygy binaries aren't used
  # elsewhere.
  if sys.platform not in ('win32', 'cygwin'):
    return

  options = _ParseCommandLine()

  if options.dry_run:
    _LOGGER.debug('Performing a dry-run.')

  # Load the current installation state, and validate it against the
  # requested installation.
  state, is_consistent = _GetCurrentState(options.revision, options.output_dir)

  # Decide whether or not an install is necessary.
  if options.force:
    _LOGGER.debug('Forcing reinstall of binaries.')
  elif is_consistent:
    # Avoid doing any work if the contents of the directory are consistent.
    _LOGGER.debug('State unchanged, no reinstall necessary.')
    return

  # Under normal logging this is the only only message that will be reported.
  _LOGGER.info('Installing revision %s Syzygy binaries.',
               options.revision[0:12])

  # Clean up the old state to begin with.
  deleted = []
  if options.overwrite:
    if os.path.exists(options.output_dir):
      # If overwrite was specified then take a heavy-handed approach.
      _LOGGER.debug('Deleting entire installation directory.')
      if not options.dry_run:
        _RmTree(options.output_dir)
  else:
    # Otherwise only delete things that the previous installation put in place,
    # and take care to preserve any local changes.
    deleted = _CleanState(options.output_dir, state, options.dry_run)

  # Install the new binaries. In a dry-run this will actually download the
  # archives, but it won't write anything to disk.
  state = _InstallBinaries(options, deleted)

  # Build and save the state for the directory.
  _SaveState(options.output_dir, state, options.dry_run)


if __name__ == '__main__':
  main()