From dd7adf72ffb9886d7cba065b6c40c2941081b37b Mon Sep 17 00:00:00 2001 From: "phajdan.jr" Date: Tue, 1 Sep 2015 08:58:51 -0700 Subject: runtest.py: remove code that tries to clean up temporary files This shouldn't be needed now and helps simplify the code. BUG=506498 Review URL: https://codereview.chromium.org/1298463004 Cr-Commit-Position: refs/heads/master@{#346666} --- .../legacy/scripts/common/chromium_utils.py | 95 ------------- infra/scripts/legacy/scripts/slave/runtest.py | 4 - infra/scripts/legacy/scripts/slave/slave_utils.py | 153 --------------------- 3 files changed, 252 deletions(-) (limited to 'infra/scripts') diff --git a/infra/scripts/legacy/scripts/common/chromium_utils.py b/infra/scripts/legacy/scripts/common/chromium_utils.py index 3aa1e2b..c4093fb 100644 --- a/infra/scripts/legacy/scripts/common/chromium_utils.py +++ b/infra/scripts/legacy/scripts/common/chromium_utils.py @@ -46,101 +46,6 @@ def IsMac(): return sys.platform.startswith('darwin') -def RemoveFile(*path): - """Removes the file located at 'path', if it exists.""" - file_path = os.path.join(*path) - try: - os.remove(file_path) - except OSError, e: - if e.errno != errno.ENOENT: - raise - - -def RemoveDirectory(*path): - """Recursively removes a directory, even if it's marked read-only. - - Remove the directory located at *path, if it exists. - - shutil.rmtree() doesn't work on Windows if any of the files or directories - are read-only, which svn repositories and some .svn files are. We need to - be able to force the files to be writable (i.e., deletable) as we traverse - the tree. - - Even with all this, Windows still sometimes fails to delete a file, citing - a permission error (maybe something to do with antivirus scans or disk - indexing). The best suggestion any of the user forums had was to wait a - bit and try again, so we do that too. It's hand-waving, but sometimes it - works. :/ - """ - file_path = os.path.join(*path) - if not os.path.exists(file_path): - return - - if sys.platform == 'win32': - # Give up and use cmd.exe's rd command. - file_path = os.path.normcase(file_path) - for _ in xrange(3): - print 'RemoveDirectory running %s' % (' '.join( - ['cmd.exe', '/c', 'rd', '/q', '/s', file_path])) - if not subprocess.call(['cmd.exe', '/c', 'rd', '/q', '/s', file_path]): - break - print ' Failed' - time.sleep(3) - return - - def RemoveWithRetry_non_win(rmfunc, path): - if os.path.islink(path): - return os.remove(path) - else: - return rmfunc(path) - - remove_with_retry = RemoveWithRetry_non_win - - def RmTreeOnError(function, path, excinfo): - r"""This works around a problem whereby python 2.x on Windows has no ability - to check for symbolic links. os.path.islink always returns False. But - shutil.rmtree will fail if invoked on a symbolic link whose target was - deleted before the link. E.g., reproduce like this: - > mkdir test - > mkdir test\1 - > mklink /D test\current test\1 - > python -c "import chromium_utils; chromium_utils.RemoveDirectory('test')" - To avoid this issue, we pass this error-handling function to rmtree. If - we see the exact sort of failure, we ignore it. All other failures we re- - raise. - """ - - exception_type = excinfo[0] - exception_value = excinfo[1] - # If shutil.rmtree encounters a symbolic link on Windows, os.listdir will - # fail with a WindowsError exception with an ENOENT errno (i.e., file not - # found). We'll ignore that error. Note that WindowsError is not defined - # for non-Windows platforms, so we use OSError (of which it is a subclass) - # to avoid lint complaints about an undefined global on non-Windows - # platforms. - if (function is os.listdir) and issubclass(exception_type, OSError): - if exception_value.errno == errno.ENOENT: - # File does not exist, and we're trying to delete, so we can ignore the - # failure. - print 'WARNING: Failed to list %s during rmtree. Ignoring.\n' % path - else: - raise - else: - raise - - for root, dirs, files in os.walk(file_path, topdown=False): - # For POSIX: making the directory writable guarantees removability. - # Windows will ignore the non-read-only bits in the chmod value. - os.chmod(root, 0770) - for name in files: - remove_with_retry(os.remove, os.path.join(root, name)) - for name in dirs: - remove_with_retry(lambda p: shutil.rmtree(p, onerror=RmTreeOnError), - os.path.join(root, name)) - - remove_with_retry(os.rmdir, file_path) - - def FindUpwardParent(start_dir, *desired_list): """Finds the desired object's parent, searching upward from the start_dir. diff --git a/infra/scripts/legacy/scripts/slave/runtest.py b/infra/scripts/legacy/scripts/slave/runtest.py index 4e09d37..c5cc9d6 100755 --- a/infra/scripts/legacy/scripts/slave/runtest.py +++ b/infra/scripts/legacy/scripts/slave/runtest.py @@ -240,10 +240,6 @@ def _Main(options, args, extra_env): command = _BuildTestBinaryCommand(build_dir, test_exe_path, options) command.extend(args[1:]) - # Nuke anything that appears to be stale chrome items in the temporary - # directory from previous test runs (i.e.- from crashes or unittest leaks). - slave_utils.RemoveChromeTemporaryFiles() - log_processor = None if _UsingGtestJson(options): log_processor = gtest_utils.GTestJSONParser( diff --git a/infra/scripts/legacy/scripts/slave/slave_utils.py b/infra/scripts/legacy/scripts/slave/slave_utils.py index 3fe22f98..d7968c03 100644 --- a/infra/scripts/legacy/scripts/slave/slave_utils.py +++ b/infra/scripts/legacy/scripts/slave/slave_utils.py @@ -55,159 +55,6 @@ def SlaveBaseDir(chrome_dir): return result -def LogAndRemoveFiles(temp_dir, regex_pattern): - """Removes files in |temp_dir| that match |regex_pattern|. - This function prints out the name of each directory or filename before - it deletes the file from disk.""" - regex = re.compile(regex_pattern) - if not os.path.isdir(temp_dir): - return - for dir_item in os.listdir(temp_dir): - if regex.search(dir_item): - full_path = os.path.join(temp_dir, dir_item) - print 'Removing leaked temp item: %s' % full_path - try: - if os.path.islink(full_path) or os.path.isfile(full_path): - os.remove(full_path) - elif os.path.isdir(full_path): - chromium_utils.RemoveDirectory(full_path) - else: - print 'Temp item wasn\'t a file or directory?' - except OSError, e: - print >> sys.stderr, e - # Don't fail. - - -def RemoveOldSnapshots(desktop): - """Removes ChromiumSnapshot files more than one day old. Such snapshots are - created when certain tests timeout (e.g., Chrome Frame integration tests).""" - # Compute the file prefix of a snapshot created one day ago. - yesterday = datetime.datetime.now() - datetime.timedelta(1) - old_snapshot = yesterday.strftime('ChromiumSnapshot%Y%m%d%H%M%S') - # Collect snapshots at least as old as that one created a day ago. - to_delete = [] - for snapshot in glob.iglob(os.path.join(desktop, 'ChromiumSnapshot*.png')): - if os.path.basename(snapshot) < old_snapshot: - to_delete.append(snapshot) - # Delete the collected snapshots. - for snapshot in to_delete: - print 'Removing old snapshot: %s' % snapshot - try: - os.remove(snapshot) - except OSError, e: - print >> sys.stderr, e - - -def RemoveChromeDesktopFiles(): - """Removes Chrome files (i.e. shortcuts) from the desktop of the current user. - This does nothing if called on a non-Windows platform.""" - if chromium_utils.IsWindows(): - desktop_path = os.environ['USERPROFILE'] - desktop_path = os.path.join(desktop_path, 'Desktop') - LogAndRemoveFiles(desktop_path, r'^(Chromium|chrome) \(.+\)?\.lnk$') - RemoveOldSnapshots(desktop_path) - - -def RemoveJumpListFiles(): - """Removes the files storing jump list history. - This does nothing if called on a non-Windows platform.""" - if chromium_utils.IsWindows(): - custom_destination_path = os.path.join(os.environ['USERPROFILE'], - 'AppData', - 'Roaming', - 'Microsoft', - 'Windows', - 'Recent', - 'CustomDestinations') - LogAndRemoveFiles(custom_destination_path, '.+') - - -def RemoveTempDirContents(): - """Obliterate the entire contents of the temporary directory, excluding - paths in sys.argv. - """ - temp_dir = os.path.abspath(tempfile.gettempdir()) - print 'Removing contents of %s' % temp_dir - - print ' Inspecting args for files to skip' - whitelist = set() - for i in sys.argv: - try: - if '=' in i: - i = i.split('=')[1] - low = os.path.abspath(i.lower()) - if low.startswith(temp_dir.lower()): - whitelist.add(low) - except TypeError: - # If the argument is too long, windows will freak out and pop a TypeError. - pass - if whitelist: - print ' Whitelisting:' - for w in whitelist: - print ' %r' % w - - start_time = time.time() - for root, dirs, files in os.walk(temp_dir): - for f in files: - p = os.path.join(root, f) - if p.lower() not in whitelist: - try: - os.remove(p) - except OSError: - pass - else: - print ' Keeping file %r (whitelisted)' % p - for d in dirs[:]: - p = os.path.join(root, d) - if p.lower() not in whitelist: - try: - # TODO(iannucci): Make this deal with whitelisted items which are - # inside of |d| - - # chromium_utils.RemoveDirectory gives access denied error when called - # in this loop. - shutil.rmtree(p, ignore_errors=True) - # Remove it so that os.walk() doesn't try to recurse into - # a non-existing directory. - dirs.remove(d) - except OSError: - pass - else: - print ' Keeping dir %r (whitelisted)' % p - print ' Removing temp contents took %.1f s' % (time.time() - start_time) - - -def RemoveChromeTemporaryFiles(): - """A large hammer to nuke what could be leaked files from unittests or - files left from a unittest that crashed, was killed, etc.""" - # NOTE: print out what is cleaned up so the bots don't timeout if - # there is a lot to cleanup and also se we see the leaks in the - # build logs. - # At some point a leading dot got added, support with and without it. - kLogRegex = r'^\.?(com\.google\.Chrome|org\.chromium)\.' - if chromium_utils.IsWindows(): - RemoveTempDirContents() - RemoveChromeDesktopFiles() - RemoveJumpListFiles() - elif chromium_utils.IsLinux(): - LogAndRemoveFiles(tempfile.gettempdir(), kLogRegex) - LogAndRemoveFiles('/dev/shm', kLogRegex) - elif chromium_utils.IsMac(): - nstempdir_path = '/usr/local/libexec/nstempdir' - if os.path.exists(nstempdir_path): - ns_temp_dir = subprocess.check_output([nstempdir_path]).strip() - if ns_temp_dir: - LogAndRemoveFiles(ns_temp_dir, kLogRegex) - for i in ('Chromium', 'Google Chrome'): - # Remove dumps. - crash_path = '%s/Library/Application Support/%s/Crash Reports' % ( - os.environ['HOME'], i) - LogAndRemoveFiles(crash_path, r'^.+\.dmp$') - else: - raise NotImplementedError( - 'Platform "%s" is not currently supported.' % sys.platform) - - def WriteLogLines(logname, lines, perf=None): logname = logname.rstrip() lines = [line.rstrip() for line in lines] -- cgit v1.1