1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
|
#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Script that attempts to push to a special git repository to verify that git
credentials are configured correctly. It also verifies that gclient solution is
configured to use git checkout.
It will be added as gclient hook shortly before Chromium switches to git and
removed after the switch.
When running as hook in *.corp.google.com network it will also report status
of the push attempt to the server (on appengine), so that chrome-infra team can
collect information about misconfigured Git accounts.
"""
import contextlib
import datetime
import errno
import getpass
import json
import logging
import netrc
import optparse
import os
import pprint
import shutil
import socket
import ssl
import subprocess
import sys
import tempfile
import time
import urllib2
import urlparse
# Absolute path to src/ directory.
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Absolute path to a file with gclient solutions.
GCLIENT_CONFIG = os.path.join(os.path.dirname(REPO_ROOT), '.gclient')
# Incremented whenever some changes to scrip logic are made. Change in version
# will cause the check to be rerun on next gclient runhooks invocation.
CHECKER_VERSION = 1
# Do not attempt to upload a report after this date.
UPLOAD_DISABLE_TS = datetime.datetime(2014, 10, 1)
# URL to POST json with results to.
MOTHERSHIP_URL = (
'https://chromium-git-access.appspot.com/'
'git_access/api/v1/reports/access_check')
# Repository to push test commits to.
TEST_REPO_URL = 'https://chromium.googlesource.com/a/playground/access_test'
# Git-compatible gclient solution.
GOOD_GCLIENT_SOLUTION = {
'name': 'src',
'deps_file': 'DEPS',
'managed': False,
'url': 'https://chromium.googlesource.com/chromium/src.git',
}
# Possible chunks of git push response in case .netrc is misconfigured.
BAD_ACL_ERRORS = (
'(prohibited by Gerrit)',
'does not match your user account',
'Git repository not found',
'Invalid user name or password',
'Please make sure you have the correct access rights',
)
# Git executable to call.
GIT_EXE = 'git.bat' if sys.platform == 'win32' else 'git'
def is_on_bot():
"""True when running under buildbot."""
return os.environ.get('CHROME_HEADLESS') == '1'
def is_in_google_corp():
"""True when running in google corp network."""
try:
return socket.getfqdn().endswith('.corp.google.com')
except socket.error:
logging.exception('Failed to get FQDN')
return False
def is_using_git():
"""True if git checkout is used."""
return os.path.exists(os.path.join(REPO_ROOT, '.git', 'objects'))
def is_using_svn():
"""True if svn checkout is used."""
return os.path.exists(os.path.join(REPO_ROOT, '.svn'))
def read_git_config(prop):
"""Reads git config property of src.git repo.
Returns empty string in case of errors.
"""
try:
proc = subprocess.Popen(
[GIT_EXE, 'config', prop], stdout=subprocess.PIPE, cwd=REPO_ROOT)
out, _ = proc.communicate()
return out.strip().decode('utf-8')
except OSError as exc:
if exc.errno != errno.ENOENT:
logging.exception('Unexpected error when calling git')
return ''
def read_netrc_user(netrc_obj, host):
"""Reads 'user' field of a host entry in netrc.
Returns empty string if netrc is missing, or host is not there.
"""
if not netrc_obj:
return ''
entry = netrc_obj.authenticators(host)
if not entry:
return ''
return entry[0]
def get_git_version():
"""Returns version of git or None if git is not available."""
try:
proc = subprocess.Popen([GIT_EXE, '--version'], stdout=subprocess.PIPE)
out, _ = proc.communicate()
return out.strip() if proc.returncode == 0 else ''
except OSError as exc:
if exc.errno != errno.ENOENT:
logging.exception('Unexpected error when calling git')
return ''
def read_gclient_solution():
"""Read information about 'src' gclient solution from .gclient file.
Returns tuple:
(url, deps_file, managed)
or
(None, None, None) if no such solution.
"""
try:
env = {}
execfile(GCLIENT_CONFIG, env, env)
for sol in (env.get('solutions') or []):
if sol.get('name') == 'src':
return sol.get('url'), sol.get('deps_file'), sol.get('managed')
return None, None, None
except Exception:
logging.exception('Failed to read .gclient solution')
return None, None, None
def read_git_insteadof(host):
"""Reads relevant insteadOf config entries."""
try:
proc = subprocess.Popen([GIT_EXE, 'config', '-l'], stdout=subprocess.PIPE)
out, _ = proc.communicate()
lines = []
for line in out.strip().split('\n'):
line = line.lower()
if 'insteadof=' in line and host in line:
lines.append(line)
return '\n'.join(lines)
except OSError as exc:
if exc.errno != errno.ENOENT:
logging.exception('Unexpected error when calling git')
return ''
def scan_configuration():
"""Scans local environment for git related configuration values."""
# Git checkout?
is_git = is_using_git()
# On Windows HOME should be set.
if 'HOME' in os.environ:
netrc_path = os.path.join(
os.environ['HOME'],
'_netrc' if sys.platform.startswith('win') else '.netrc')
else:
netrc_path = None
# Netrc exists?
is_using_netrc = netrc_path and os.path.exists(netrc_path)
# Read it.
netrc_obj = None
if is_using_netrc:
try:
netrc_obj = netrc.netrc(netrc_path)
except Exception:
logging.exception('Failed to read netrc from %s', netrc_path)
netrc_obj = None
# Read gclient 'src' solution.
gclient_url, gclient_deps, gclient_managed = read_gclient_solution()
return {
'checker_version': CHECKER_VERSION,
'is_git': is_git,
'is_home_set': 'HOME' in os.environ,
'is_using_netrc': is_using_netrc,
'netrc_file_mode': os.stat(netrc_path).st_mode if is_using_netrc else 0,
'git_version': get_git_version(),
'platform': sys.platform,
'username': getpass.getuser(),
'git_user_email': read_git_config('user.email') if is_git else '',
'git_user_name': read_git_config('user.name') if is_git else '',
'git_insteadof': read_git_insteadof('chromium.googlesource.com'),
'chromium_netrc_email':
read_netrc_user(netrc_obj, 'chromium.googlesource.com'),
'chrome_internal_netrc_email':
read_netrc_user(netrc_obj, 'chrome-internal.googlesource.com'),
'gclient_deps': gclient_deps,
'gclient_managed': gclient_managed,
'gclient_url': gclient_url,
}
def last_configuration_path():
"""Path to store last checked configuration."""
if is_using_git():
return os.path.join(REPO_ROOT, '.git', 'check_git_push_access_conf.json')
elif is_using_svn():
return os.path.join(REPO_ROOT, '.svn', 'check_git_push_access_conf.json')
else:
return os.path.join(REPO_ROOT, '.check_git_push_access_conf.json')
def read_last_configuration():
"""Reads last checked configuration if it exists."""
try:
with open(last_configuration_path(), 'r') as f:
return json.load(f)
except (IOError, ValueError):
return None
def write_last_configuration(conf):
"""Writes last checked configuration to a file."""
try:
with open(last_configuration_path(), 'w') as f:
json.dump(conf, f, indent=2, sort_keys=True)
except IOError:
logging.exception('Failed to write JSON to %s', path)
@contextlib.contextmanager
def temp_directory():
"""Creates a temp directory, then nukes it."""
tmp = tempfile.mkdtemp()
try:
yield tmp
finally:
try:
shutil.rmtree(tmp)
except (OSError, IOError):
logging.exception('Failed to remove temp directory %s', tmp)
class Runner(object):
"""Runs a bunch of commands in some directory, collects logs from them."""
def __init__(self, cwd, verbose):
self.cwd = cwd
self.verbose = verbose
self.log = []
def run(self, cmd):
self.append_to_log('> ' + ' '.join(cmd))
retcode = -1
try:
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=self.cwd)
out, _ = proc.communicate()
out = out.strip()
retcode = proc.returncode
except OSError as exc:
out = str(exc)
if retcode:
out += '\n(exit code: %d)' % retcode
self.append_to_log(out)
return retcode
def append_to_log(self, text):
if text:
self.log.append(text)
if self.verbose:
logging.warning(text)
def check_git_config(conf, report_url, verbose):
"""Attempts to push to a git repository, reports results to a server.
Returns True if the check finished without incidents (push itself may
have failed) and should NOT be retried on next invocation of the hook.
"""
# Don't even try to push if netrc is not configured.
if not conf['chromium_netrc_email']:
return upload_report(
conf,
report_url,
verbose,
push_works=False,
push_log='',
push_duration_ms=0)
# Ref to push to, each user has its own ref.
ref = 'refs/push-test/%s' % conf['chromium_netrc_email']
push_works = False
flake = False
started = time.time()
try:
logging.warning('Checking push access to the git repository...')
with temp_directory() as tmp:
# Prepare a simple commit on a new timeline.
runner = Runner(tmp, verbose)
runner.run([GIT_EXE, 'init', '.'])
if conf['git_user_name']:
runner.run([GIT_EXE, 'config', 'user.name', conf['git_user_name']])
if conf['git_user_email']:
runner.run([GIT_EXE, 'config', 'user.email', conf['git_user_email']])
with open(os.path.join(tmp, 'timestamp'), 'w') as f:
f.write(str(int(time.time() * 1000)))
runner.run([GIT_EXE, 'add', 'timestamp'])
runner.run([GIT_EXE, 'commit', '-m', 'Push test.'])
# Try to push multiple times if it fails due to issues other than ACLs.
attempt = 0
while attempt < 5:
attempt += 1
logging.info('Pushing to %s %s', TEST_REPO_URL, ref)
ret = runner.run(
[GIT_EXE, 'push', TEST_REPO_URL, 'HEAD:%s' % ref, '-f'])
if not ret:
push_works = True
break
if any(x in runner.log[-1] for x in BAD_ACL_ERRORS):
push_works = False
break
except Exception:
logging.exception('Unexpected exception when pushing')
flake = True
if push_works:
logging.warning('Git push works!')
else:
logging.warning(
'Git push doesn\'t work, which is fine if you are not a committer.')
uploaded = upload_report(
conf,
report_url,
verbose,
push_works=push_works,
push_log='\n'.join(runner.log),
push_duration_ms=int((time.time() - started) * 1000))
return uploaded and not flake
def check_gclient_config(conf):
"""Shows warning if gclient solution is not properly configured for git."""
# Ignore configs that do not have 'src' solution at all.
if not conf['gclient_url']:
return
current = {
'name': 'src',
'deps_file': conf['gclient_deps'] or 'DEPS',
'managed': conf['gclient_managed'] or False,
'url': conf['gclient_url'],
}
# After depot_tools r291592 both DEPS and .DEPS.git are valid.
good = GOOD_GCLIENT_SOLUTION.copy()
good['deps_file'] = current['deps_file']
if current == good:
return
# Show big warning if url or deps_file is wrong.
if current['url'] != good['url'] or current['deps_file'] != good['deps_file']:
print '-' * 80
print 'Your gclient solution is not set to use supported git workflow!'
print
print 'Your \'src\' solution (in %s):' % GCLIENT_CONFIG
print pprint.pformat(current, indent=2)
print
print 'Correct \'src\' solution to use git:'
print pprint.pformat(good, indent=2)
print
print 'Please update your .gclient file ASAP.'
print '-' * 80
# Show smaller (additional) warning about managed workflow.
if current['managed']:
print '-' * 80
print (
'You are using managed gclient mode with git, which was deprecated '
'on 8/22/13:')
print (
'https://groups.google.com/a/chromium.org/'
'forum/#!topic/chromium-dev/n9N5N3JL2_U')
print
print (
'It is strongly advised to switch to unmanaged mode. For more '
'information about managed mode and reasons for its deprecation see:')
print 'http://www.chromium.org/developers/how-tos/get-the-code/gclient-managed-mode'
print
print (
'There\'s also a large suite of tools to assist managing git '
'checkouts.\nSee \'man depot_tools\' (or read '
'depot_tools/man/html/depot_tools.html).')
print '-' * 80
def upload_report(
conf, report_url, verbose, push_works, push_log, push_duration_ms):
"""Posts report to the server, returns True if server accepted it.
Uploads the report only if script is running in Google corp network. Otherwise
just prints the report.
"""
report = conf.copy()
report.update(
push_works=push_works,
push_log=push_log,
push_duration_ms=push_duration_ms)
as_bytes = json.dumps({'access_check': report}, indent=2, sort_keys=True)
if verbose:
print 'Status of git push attempt:'
print as_bytes
# Do not upload it outside of corp or if server side is already disabled.
if not is_in_google_corp() or datetime.datetime.now() > UPLOAD_DISABLE_TS:
if verbose:
print (
'You can send the above report to chrome-git-migration@google.com '
'if you need help to set up you committer git account.')
return True
req = urllib2.Request(
url=report_url,
data=as_bytes,
headers={'Content-Type': 'application/json; charset=utf-8'})
attempt = 0
success = False
while not success and attempt < 10:
attempt += 1
try:
logging.warning(
'Attempting to upload the report to %s...',
urlparse.urlparse(report_url).netloc)
resp = urllib2.urlopen(req, timeout=5)
report_id = None
try:
report_id = json.load(resp)['report_id']
except (ValueError, TypeError, KeyError):
pass
logging.warning('Report uploaded: %s', report_id)
success = True
except (urllib2.URLError, socket.error, ssl.SSLError) as exc:
logging.warning('Failed to upload the report: %s', exc)
return success
def main(args):
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option(
'--running-as-hook',
action='store_true',
help='Set when invoked from gclient hook')
parser.add_option(
'--report-url',
default=MOTHERSHIP_URL,
help='URL to submit the report to')
parser.add_option(
'--verbose',
action='store_true',
help='More logging')
options, args = parser.parse_args()
if args:
parser.error('Unknown argument %s' % args)
logging.basicConfig(
format='%(message)s',
level=logging.INFO if options.verbose else logging.WARN)
# When invoked not as a hook, always run the check.
if not options.running_as_hook:
config = scan_configuration()
check_gclient_config(config)
check_git_config(config, options.report_url, True)
return 0
# Always do nothing on bots.
if is_on_bot():
return 0
# Read current config, verify gclient solution looks correct.
config = scan_configuration()
check_gclient_config(config)
# Do not attempt to push from non-google owned machines.
if not is_in_google_corp():
logging.info('Skipping git push check: non *.corp.google.com machine.')
return 0
# Skip git push check if current configuration was already checked.
if config == read_last_configuration():
logging.info('Check already performed, skipping.')
return 0
# Run the check. Mark configuration as checked only on success. Ignore any
# exceptions or errors. This check must not break gclient runhooks.
try:
ok = check_git_config(config, options.report_url, False)
if ok:
write_last_configuration(config)
else:
logging.warning('Check failed and will be retried on the next run')
except Exception:
logging.exception('Unexpected exception when performing git access check')
return 0
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))
|