summaryrefslogtreecommitdiffstats
path: root/tools/site_compare
diff options
context:
space:
mode:
authormaruel@chromium.org <maruel@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>2009-03-05 12:46:38 +0000
committermaruel@chromium.org <maruel@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>2009-03-05 12:46:38 +0000
commitf0a51fb571f46531025fa09240bbc3e1af925e84 (patch)
tree558b4f0e737fda4b9ab60f252c9c23b8a4ca523e /tools/site_compare
parent6390be368205705f49ead3cec40396519f13b889 (diff)
downloadchromium_src-f0a51fb571f46531025fa09240bbc3e1af925e84.zip
chromium_src-f0a51fb571f46531025fa09240bbc3e1af925e84.tar.gz
chromium_src-f0a51fb571f46531025fa09240bbc3e1af925e84.tar.bz2
Fixes CRLF and trailing white spaces.
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@10982 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'tools/site_compare')
-rw-r--r--tools/site_compare/command_line.py292
-rw-r--r--tools/site_compare/commands/compare2.py38
-rw-r--r--tools/site_compare/commands/maskmaker.py46
-rw-r--r--tools/site_compare/commands/measure.py4
-rw-r--r--tools/site_compare/commands/scrape.py6
-rw-r--r--tools/site_compare/commands/timeload.py38
-rw-r--r--tools/site_compare/drivers/__init__.py2
-rw-r--r--tools/site_compare/drivers/win32/keyboard.py66
-rw-r--r--tools/site_compare/drivers/win32/mouse.py80
-rw-r--r--tools/site_compare/drivers/win32/windowing.py110
-rw-r--r--tools/site_compare/operators/__init__.py10
-rw-r--r--tools/site_compare/operators/equals.py16
-rw-r--r--tools/site_compare/operators/equals_with_mask.py26
-rw-r--r--tools/site_compare/scrapers/__init__.py12
-rw-r--r--tools/site_compare/scrapers/chrome/__init__.py14
-rw-r--r--tools/site_compare/scrapers/chrome/chrome011010.py8
-rw-r--r--tools/site_compare/scrapers/chrome/chrome01970.py12
-rw-r--r--tools/site_compare/scrapers/chrome/chromebase.py70
-rw-r--r--tools/site_compare/scrapers/firefox/__init__.py16
-rw-r--r--tools/site_compare/scrapers/firefox/firefox2.py86
-rw-r--r--tools/site_compare/scrapers/ie/__init__.py12
-rw-r--r--tools/site_compare/scrapers/ie/ie7.py66
-rw-r--r--tools/site_compare/site_compare.py58
-rw-r--r--tools/site_compare/utils/browser_iterate.py22
24 files changed, 555 insertions, 555 deletions
diff --git a/tools/site_compare/command_line.py b/tools/site_compare/command_line.py
index b474abf..2c87fb9 100644
--- a/tools/site_compare/command_line.py
+++ b/tools/site_compare/command_line.py
@@ -56,11 +56,11 @@ class Command(object):
"""Encapsulates an argument to a command."""
VALID_TYPES = ['string', 'readfile', 'int', 'flag', 'coords']
TYPES_WITH_VALUES = ['string', 'readfile', 'int', 'coords']
-
+
def __init__(self, names, helptext, type, metaname,
required, default, positional):
"""Command-line argument to a command.
-
+
Args:
names: argument name, or list of synonyms
helptext: brief description of the argument
@@ -76,28 +76,28 @@ class Command(object):
required: True if argument must be specified
default: Default value if not specified
positional: Argument specified by location, not name
-
+
Raises:
ValueError: the argument name is invalid for some reason
"""
if type not in Command.Argument.VALID_TYPES:
raise ValueError("Invalid type: %r" % type)
-
+
if required and default is not None:
raise ValueError("required and default are mutually exclusive")
-
+
if required and type == 'flag':
raise ValueError("A required flag? Give me a break.")
-
+
if metaname and type not in Command.Argument.TYPES_WITH_VALUES:
raise ValueError("Type %r can't have a metaname" % type)
-
+
# If no metaname is provided, infer it: use the alphabetical characters
# of the last provided name
if not metaname and type in Command.Argument.TYPES_WITH_VALUES:
metaname = (
names[-1].lstrip(string.punctuation + string.whitespace).upper())
-
+
self.names = names
self.helptext = helptext
self.type = type
@@ -105,31 +105,31 @@ class Command(object):
self.default = default
self.positional = positional
self.metaname = metaname
-
+
self.mutex = [] # arguments that are mutually exclusive with
# this one
self.depends = [] # arguments that must be present for this
# one to be valid
self.present = False # has this argument been specified?
-
+
def AddDependency(self, arg):
"""Makes this argument dependent on another argument.
-
+
Args:
arg: name of the argument this one depends on
"""
if arg not in self.depends:
self.depends.append(arg)
-
+
def AddMutualExclusion(self, arg):
"""Makes this argument invalid if another is specified.
-
+
Args:
arg: name of the mutually exclusive argument.
"""
if arg not in self.mutex:
self.mutex.append(arg)
-
+
def GetUsageString(self):
"""Returns a brief string describing the argument's usage."""
if not self.positional:
@@ -138,49 +138,49 @@ class Command(object):
string += "="+self.metaname
else:
string = self.metaname
-
+
if not self.required:
string = "["+string+"]"
-
+
return string
-
+
def GetNames(self):
"""Returns a string containing a list of the arg's names."""
if self.positional:
return self.metaname
else:
return ", ".join(self.names)
-
+
def GetHelpString(self, width=80, indent=5, names_width=20, gutter=2):
"""Returns a help string including help for all the arguments."""
names = [" "*indent + line +" "*(names_width-len(line)) for line in
textwrap.wrap(self.GetNames(), names_width)]
-
+
helpstring = textwrap.wrap(self.helptext, width-indent-names_width-gutter)
-
+
if len(names) < len(helpstring):
names += [" "*(indent+names_width)]*(len(helpstring)-len(names))
-
+
if len(helpstring) < len(names):
helpstring += [""]*(len(names)-len(helpstring))
-
+
return "\n".join([name_line + " "*gutter + help_line for
name_line, help_line in zip(names, helpstring)])
-
+
def __repr__(self):
if self.present:
string = '= %r' % self.value
else:
string = "(absent)"
-
+
return "Argument %s '%s'%s" % (self.type, self.names[0], string)
-
+
# end of nested class Argument
-
+
def AddArgument(self, names, helptext, type="string", metaname=None,
required=False, default=None, positional=False):
"""Command-line argument to a command.
-
+
Args:
names: argument name, or list of synonyms
helptext: brief description of the argument
@@ -189,82 +189,82 @@ class Command(object):
required: True if argument must be specified
default: Default value if not specified
positional: Argument specified by location, not name
-
+
Raises:
ValueError: the argument already exists or is invalid
-
+
Returns:
The newly-created argument
"""
if IsString(names): names = [names]
-
+
names = [name.lower() for name in names]
-
+
for name in names:
if name in self.arg_dict:
raise ValueError("%s is already an argument"%name)
-
+
if (positional and required and
[arg for arg in self.args if arg.positional] and
not [arg for arg in self.args if arg.positional][-1].required):
raise ValueError(
"A required positional argument may not follow an optional one.")
-
+
arg = Command.Argument(names, helptext, type, metaname,
required, default, positional)
-
+
self.args.append(arg)
-
+
for name in names:
self.arg_dict[name] = arg
-
+
return arg
-
+
def GetArgument(self, name):
"""Return an argument from a name."""
return self.arg_dict[name.lower()]
-
+
def AddMutualExclusion(self, args):
"""Specifies that a list of arguments are mutually exclusive."""
if len(args) < 2:
raise ValueError("At least two arguments must be specified.")
-
+
args = [arg.lower() for arg in args]
-
+
for index in xrange(len(args)-1):
for index2 in xrange(index+1, len(args)):
self.arg_dict[args[index]].AddMutualExclusion(self.arg_dict[args[index2]])
-
+
def AddDependency(self, dependent, depends_on):
"""Specifies that one argument may only be present if another is.
-
+
Args:
dependent: the name of the dependent argument
depends_on: the name of the argument on which it depends
"""
self.arg_dict[dependent.lower()].AddDependency(
self.arg_dict[depends_on.lower()])
-
+
def AddMutualDependency(self, args):
"""Specifies that a list of arguments are all mutually dependent."""
if len(args) < 2:
raise ValueError("At least two arguments must be specified.")
-
+
args = [arg.lower() for arg in args]
-
+
for (arg1, arg2) in [(arg1, arg2) for arg1 in args for arg2 in args]:
if arg1 == arg2: continue
self.arg_dict[arg1].AddDependency(self.arg_dict[arg2])
-
+
def AddRequiredGroup(self, args):
"""Specifies that at least one of the named arguments must be present."""
if len(args) < 2:
raise ValueError("At least two arguments must be in a required group.")
-
+
args = [self.arg_dict[arg.lower()] for arg in args]
-
+
self.required_groups.append(args)
-
+
def ParseArguments(self):
"""Given a command line, parse and validate the arguments."""
@@ -272,70 +272,70 @@ class Command(object):
for arg in self.args:
arg.present = False
arg.value = None
-
+
self.parse_errors = []
-
+
# look for arguments remaining on the command line
while len(self.cmdline.rargs):
try:
self.ParseNextArgument()
except ParseError, e:
self.parse_errors.append(e.args[0])
-
+
# after all the arguments are parsed, check for problems
for arg in self.args:
if not arg.present and arg.required:
self.parse_errors.append("'%s': required parameter was missing"
% arg.names[0])
-
+
if not arg.present and arg.default:
arg.present = True
arg.value = arg.default
-
+
if arg.present:
for mutex in arg.mutex:
if mutex.present:
self.parse_errors.append(
"'%s', '%s': arguments are mutually exclusive" %
(arg.argstr, mutex.argstr))
-
+
for depend in arg.depends:
if not depend.present:
self.parse_errors.append("'%s': '%s' must be specified as well" %
(arg.argstr, depend.names[0]))
-
+
# check for required groups
for group in self.required_groups:
if not [arg for arg in group if arg.present]:
self.parse_errors.append("%s: at least one must be present" %
(", ".join(["'%s'" % arg.names[-1] for arg in group])))
-
+
# if we have any validators, invoke them
if not self.parse_errors and self.validator:
try:
self.validator(self)
except ParseError, e:
self.parse_errors.append(e.args[0])
-
+
# Helper methods so you can treat the command like a dict
def __getitem__(self, key):
arg = self.arg_dict[key.lower()]
-
+
if arg.type == 'flag':
return arg.present
else:
return arg.value
-
+
def __iter__(self):
return [arg for arg in self.args if arg.present].__iter__()
-
+
def ArgumentPresent(self, key):
"""Tests if an argument exists and has been specified."""
return key.lower() in self.arg_dict and self.arg_dict[key.lower()].present
-
+
def __contains__(self, key):
return self.ArgumentPresent(key)
-
+
def ParseNextArgument(self):
"""Find the next argument in the command line and parse it."""
arg = None
@@ -348,26 +348,26 @@ class Command(object):
if arg.type in Command.Argument.TYPES_WITH_VALUES:
if len(self.cmdline.rargs):
value = self.cmdline.rargs.pop(0)
-
+
# Second check: is this of the form "arg=val" or "arg:val"?
if arg is None:
delimiter_pos = -1
-
+
for delimiter in [':', '=']:
pos = argstr.find(delimiter)
if pos >= 0:
if delimiter_pos < 0 or pos < delimiter_pos:
delimiter_pos = pos
-
+
if delimiter_pos >= 0:
testarg = argstr[:delimiter_pos]
testval = argstr[delimiter_pos+1:]
-
+
if testarg.lower() in self.arg_dict:
arg = self.arg_dict[testarg.lower()]
argstr = testarg
value = testval
-
+
# Third check: does this begin an argument?
if arg is None:
for key in self.arg_dict.iterkeys():
@@ -377,7 +377,7 @@ class Command(object):
value = argstr[len(key):]
argstr = argstr[:len(key)]
arg = self.arg_dict[argstr]
-
+
# Fourth check: do we have any positional arguments available?
if arg is None:
for positional_arg in [
@@ -391,40 +391,40 @@ class Command(object):
# Push the retrieved argument/value onto the largs stack
if argstr: self.cmdline.largs.append(argstr)
if value: self.cmdline.largs.append(value)
-
+
# If we've made it this far and haven't found an arg, give up
if arg is None:
raise ParseError("Unknown argument: '%s'" % argstr)
-
+
# Convert the value, if necessary
if arg.type in Command.Argument.TYPES_WITH_VALUES and value is None:
raise ParseError("Argument '%s' requires a value" % argstr)
-
+
if value is not None:
value = self.StringToValue(value, arg.type, argstr)
arg.argstr = argstr
arg.value = value
arg.present = True
-
+
# end method ParseNextArgument
-
+
def StringToValue(self, value, type, argstr):
"""Convert a string from the command line to a value type."""
try:
if type == 'string':
pass # leave it be
-
+
elif type == 'int':
try:
value = int(value)
except ValueError:
raise ParseError
-
+
elif type == 'readfile':
if not os.path.isfile(value):
raise ParseError("'%s': '%s' does not exist" % (argstr, value))
-
+
elif type == 'coords':
try:
value = [int(val) for val in
@@ -432,10 +432,10 @@ class Command(object):
groups()]
except AttributeError:
raise ParseError
-
+
else:
raise ValueError("Unknown type: '%s'" % type)
-
+
except ParseError, e:
# The bare exception is raised in the generic case; more specific errors
# will arrive with arguments and should just be reraised
@@ -443,23 +443,23 @@ class Command(object):
e = ParseError("'%s': unable to convert '%s' to type '%s'" %
(argstr, value, type))
raise e
-
+
return value
-
+
def SortArgs(self):
"""Returns a method that can be passed to sort() to sort arguments."""
-
+
def ArgSorter(arg1, arg2):
"""Helper for sorting arguments in the usage string.
-
+
Positional arguments come first, then required arguments,
then optional arguments. Pylint demands this trivial function
have both Args: and Returns: sections, sigh.
-
+
Args:
arg1: the first argument to compare
arg2: the second argument to compare
-
+
Returns:
-1 if arg1 should be sorted first, +1 if it should be sorted second,
and 0 if arg1 and arg2 have the same sort level.
@@ -467,56 +467,56 @@ class Command(object):
return ((arg2.positional-arg1.positional)*2 +
(arg2.required-arg1.required))
return ArgSorter
-
+
def GetUsageString(self, width=80, name=None):
"""Gets a string describing how the command is used."""
if name is None: name = self.names[0]
-
+
initial_indent = "Usage: %s %s " % (self.cmdline.prog, name)
subsequent_indent = " " * len(initial_indent)
-
+
sorted_args = self.args[:]
sorted_args.sort(self.SortArgs())
-
+
return textwrap.fill(
" ".join([arg.GetUsageString() for arg in sorted_args]), width,
initial_indent=initial_indent,
subsequent_indent=subsequent_indent)
-
+
def GetHelpString(self, width=80):
"""Returns a list of help strings for all this command's arguments."""
sorted_args = self.args[:]
sorted_args.sort(self.SortArgs())
-
+
return "\n".join([arg.GetHelpString(width) for arg in sorted_args])
-
+
# end class Command
-
-
+
+
class CommandLine(object):
"""Parse a command line, extracting a command and its arguments."""
-
+
def __init__(self):
self.commands = []
self.cmd_dict = {}
-
+
# Add the help command to the parser
help_cmd = self.AddCommand(["help", "--help", "-?", "-h"],
"Displays help text for a command",
ValidateHelpCommand,
DoHelpCommand)
-
+
help_cmd.AddArgument(
"command", "Command to retrieve help for", positional=True)
help_cmd.AddArgument(
"--width", "Width of the output", type='int', default=80)
-
+
self.Exit = sys.exit # override this if you don't want the script to halt
# on error or on display of help
-
+
self.out = sys.stdout # override these if you want to redirect
self.err = sys.stderr # output or error messages
-
+
def AddCommand(self, names, helptext, validator=None, impl=None):
"""Add a new command to the parser.
@@ -525,56 +525,56 @@ class CommandLine(object):
helptext: brief string description of the command
validator: method to validate a command's arguments
impl: callable to be invoked when command is called
-
+
Raises:
ValueError: raised if command already added
-
+
Returns:
The new command
"""
if IsString(names): names = [names]
-
+
for name in names:
if name in self.cmd_dict:
raise ValueError("%s is already a command"%name)
-
+
cmd = Command(names, helptext, validator, impl)
cmd.cmdline = self
-
+
self.commands.append(cmd)
for name in names:
self.cmd_dict[name.lower()] = cmd
-
+
return cmd
-
+
def GetUsageString(self):
"""Returns simple usage instructions."""
return "Type '%s help' for usage." % self.prog
-
+
def ParseCommandLine(self, argv=None, prog=None, execute=True):
"""Does the work of parsing a command line.
-
+
Args:
argv: list of arguments, defaults to sys.args[1:]
prog: name of the command, defaults to the base name of the script
execute: if false, just parse, don't invoke the 'impl' member
-
+
Returns:
The command that was executed
"""
if argv is None: argv = sys.argv[1:]
if prog is None: prog = os.path.basename(sys.argv[0]).split('.')[0]
-
+
# Store off our parameters, we may need them someday
self.argv = argv
self.prog = prog
-
+
# We shouldn't be invoked without arguments, that's just lame
if not len(argv):
self.out.writelines(self.GetUsageString())
self.Exit()
return None # in case the client overrides Exit
-
+
# Is it a valid command?
self.command_string = argv[0].lower()
if not self.command_string in self.cmd_dict:
@@ -582,33 +582,33 @@ class CommandLine(object):
self.out.write(self.GetUsageString())
self.Exit()
return None # in case the client overrides Exit
-
+
self.command = self.cmd_dict[self.command_string]
-
+
# "rargs" = remaining (unparsed) arguments
# "largs" = already parsed, "left" of the read head
self.rargs = argv[1:]
self.largs = []
-
+
# let the command object do the parsing
self.command.ParseArguments()
-
+
if self.command.parse_errors:
# there were errors, output the usage string and exit
self.err.write(self.command.GetUsageString()+"\n\n")
self.err.write("\n".join(self.command.parse_errors))
self.err.write("\n\n")
-
+
self.Exit()
-
+
elif execute and self.command.impl:
self.command.impl(self.command)
-
+
return self.command
-
+
def __getitem__(self, key):
return self.cmd_dict[key]
-
+
def __iter__(self):
return self.cmd_dict.__iter__()
@@ -618,25 +618,25 @@ def ValidateHelpCommand(command):
if 'command' in command and command['command'] not in command.cmdline:
raise ParseError("'%s': unknown command" % command['command'])
-
+
def DoHelpCommand(command):
"""Executed when the command is 'help'."""
out = command.cmdline.out
width = command['--width']
-
+
if 'command' not in command:
out.write(command.GetUsageString())
out.write("\n\n")
-
+
indent = 5
gutter = 2
-
+
command_width = (
max([len(cmd.names[0]) for cmd in command.cmdline.commands]) + gutter)
-
+
for cmd in command.cmdline.commands:
cmd_name = cmd.names[0]
-
+
initial_indent = (" "*indent + cmd_name + " "*
(command_width+gutter-len(cmd_name)))
subsequent_indent = " "*(indent+command_width+gutter)
@@ -645,9 +645,9 @@ def DoHelpCommand(command):
initial_indent=initial_indent,
subsequent_indent=subsequent_indent))
out.write("\n")
-
+
out.write("\n")
-
+
else:
help_cmd = command.cmdline[command['command']]
@@ -657,21 +657,21 @@ def DoHelpCommand(command):
out.write("\n\n")
out.write(help_cmd.GetHelpString(width=width))
out.write("\n")
-
+
command.cmdline.Exit()
-
+
if __name__ == "__main__":
# If we're invoked rather than imported, run some tests
cmdline = CommandLine()
-
+
# Since we're testing, override Exit()
def TestExit():
pass
cmdline.Exit = TestExit
-
+
# Actually, while we're at it, let's override error output too
cmdline.err = open(os.path.devnull, "w")
-
+
test = cmdline.AddCommand(["test", "testa", "testb"], "test command")
test.AddArgument(["-i", "--int", "--integer", "--optint", "--optionalint"],
"optional integer parameter", type='int')
@@ -688,25 +688,25 @@ if __name__ == "__main__":
test.AddArgument("--mutdep2", "mutually dependent parameter 2")
test.AddArgument("--mutdep3", "mutually dependent parameter 3")
test.AddMutualDependency(["--mutdep1", "--mutdep2", "--mutdep3"])
-
+
# mutually exclusive arguments
test.AddArgument("--mutex1", "mutually exclusive parameter 1")
test.AddArgument("--mutex2", "mutually exclusive parameter 2")
test.AddArgument("--mutex3", "mutually exclusive parameter 3")
test.AddMutualExclusion(["--mutex1", "--mutex2", "--mutex3"])
-
+
# dependent argument
test.AddArgument("--dependent", "dependent argument")
test.AddDependency("--dependent", "--int")
-
+
# other argument types
test.AddArgument("--file", "filename argument", type='readfile')
test.AddArgument("--coords", "coordinate argument", type='coords')
test.AddArgument("--flag", "flag argument", type='flag')
-
+
test.AddArgument("--req1", "part of a required group", type='flag')
test.AddArgument("--req2", "part 2 of a required group", type='flag')
-
+
test.AddRequiredGroup(["--req1", "--req2"])
# a few failure cases
@@ -742,7 +742,7 @@ if __name__ == "__main__":
# Let's do some parsing! first, the minimal success line:
MIN = "test --reqint 123 param1 --req1 "
-
+
# tuples of (command line, expected error count)
test_lines = [
("test --int 3 foo --req1", 1), # missing required named parameter
@@ -781,19 +781,19 @@ if __name__ == "__main__":
(MIN+"--coords (123,456)", 0), # finally!
("test --int 123 --reqint=456 foo bar --coords(42,88) baz --req1", 0)
]
-
+
badtests = 0
-
+
for (test, expected_failures) in test_lines:
cmdline.ParseCommandLine([x.strip() for x in test.strip().split(" ")])
-
+
if not len(cmdline.command.parse_errors) == expected_failures:
print "FAILED:\n issued: '%s'\n expected: %d\n received: %d\n\n" % (
test, expected_failures, len(cmdline.command.parse_errors))
badtests += 1
-
+
print "%d failed out of %d tests" % (badtests, len(test_lines))
-
+
cmdline.ParseCommandLine(["help", "test"])
diff --git a/tools/site_compare/commands/compare2.py b/tools/site_compare/commands/compare2.py
index e970c24..045141b 100644
--- a/tools/site_compare/commands/compare2.py
+++ b/tools/site_compare/commands/compare2.py
@@ -29,7 +29,7 @@ def CreateCommand(cmdline):
"Compares the output of two browsers on the same URL or list of URLs",
ValidateCompare2,
ExecuteCompare2)
-
+
cmd.AddArgument(
["-b1", "--browser1"], "Full path to first browser's executable",
type="readfile", metaname="PATH", required=True)
@@ -81,7 +81,7 @@ def CreateCommand(cmdline):
cmd.AddArgument(
["-d", "--diffdir"], "Path to hold the difference of comparisons that fail")
-
+
def ValidateCompare2(command):
"""Validate the arguments to compare2. Raises ParseError if failed."""
executables = [".exe", ".com", ".bat"]
@@ -102,68 +102,68 @@ def ExecuteCompare2(command):
endline = command["--endline"]
url_list = [url.strip() for url in
open(command["--list"], "r").readlines()[startline:endline]]
-
+
log_file = open(command["--logfile"], "w")
outdir = command["--outdir"]
if not outdir: outdir = tempfile.gettempdir()
-
+
scrape_info_list = []
-
+
class ScrapeInfo(object):
"""Helper class to hold information about a scrape."""
__slots__ = ["browser_path", "scraper", "outdir", "result"]
-
+
for index in xrange(1, 3):
scrape_info = ScrapeInfo()
scrape_info.browser_path = command["--browser%d" % index]
scrape_info.scraper = scrapers.GetScraper(
(command["--browser"], command["--browser%dver" % index]))
-
+
if command["--browser%dname" % index]:
scrape_info.outdir = os.path.join(outdir,
command["--browser%dname" % index])
else:
scrape_info.outdir = os.path.join(outdir, str(index))
-
+
drivers.windowing.PreparePath(scrape_info.outdir)
scrape_info_list.append(scrape_info)
-
+
compare = operators.GetOperator("equals_with_mask")
-
+
for url in url_list:
success = True
-
+
for scrape_info in scrape_info_list:
scrape_info.result = scrape_info.scraper.Scrape(
[url], scrape_info.outdir, command["--size"], (0, 0),
command["--timeout"], path=scrape_info.browser_path)
-
+
if not scrape_info.result:
scrape_info.result = "success"
else:
success = False
-
+
result = "unknown"
-
+
if success:
result = "equal"
-
+
file1 = drivers.windowing.URLtoFilename(
url, scrape_info_list[0].outdir, ".bmp")
file2 = drivers.windowing.URLtoFilename(
url, scrape_info_list[1].outdir, ".bmp")
-
+
comparison_result = compare.Compare(file1, file2,
maskdir=command["--maskdir"])
-
+
if comparison_result is not None:
result = "not-equal"
-
+
if command["--diffdir"]:
comparison_result[1].save(
drivers.windowing.URLtoFilename(url, command["--diffdir"], ".bmp"))
-
+
# TODO(jhaas): maybe use the logging module rather than raw file writes
log_file.write("%s %s %s %s\n" % (url,
scrape_info_list[0].result,
diff --git a/tools/site_compare/commands/maskmaker.py b/tools/site_compare/commands/maskmaker.py
index a5bf6e4..73b732c 100644
--- a/tools/site_compare/commands/maskmaker.py
+++ b/tools/site_compare/commands/maskmaker.py
@@ -96,7 +96,7 @@ def ValidateMaskmaker(command):
def ExecuteMaskmaker(command):
"""Performs automatic mask generation."""
-
+
# Get the list of URLs to generate masks for
class MaskmakerURL(object):
"""Helper class for holding information about a URL passed to maskmaker."""
@@ -105,7 +105,7 @@ def ExecuteMaskmaker(command):
self.url = url
self.consecutive_successes = 0
self.errors = 0
-
+
if command["--url"]:
url_list = [MaskmakerURL(command["--url"])]
else:
@@ -116,22 +116,22 @@ def ExecuteMaskmaker(command):
endline = command["--endline"]
url_list = [MaskmakerURL(url.strip()) for url in
open(command["--list"], "r").readlines()[startline:endline]]
-
+
complete_list = []
error_list = []
-
+
outdir = command["--outdir"]
scrapes = command["--scrapes"]
errors = command["--errors"]
size = command["--size"]
scrape_pass = 0
-
+
scrapedir = command["--scrapedir"]
if not scrapedir: scrapedir = tempfile.gettempdir()
-
+
# Get the scraper
scraper = scrapers.GetScraper((command["--browser"], command["--browserver"]))
-
+
# Repeatedly iterate through the list of URLs until either every URL has
# a successful mask or too many errors, or we've exceeded the giveup limit
while url_list and scrape_pass < command["--giveup"]:
@@ -157,31 +157,31 @@ def ExecuteMaskmaker(command):
print " %r does not exist, creating" % mask_filename
mask = Image.new("1", size, 1)
mask.save(mask_filename)
-
+
# Find the stored scrape path
mask_scrape_dir = os.path.join(
scrapedir, os.path.splitext(os.path.basename(mask_filename))[0])
drivers.windowing.PreparePath(mask_scrape_dir)
-
+
# Find the baseline image
mask_scrapes = os.listdir(mask_scrape_dir)
mask_scrapes.sort()
-
+
if not mask_scrapes:
print " No baseline image found, mask will not be updated"
baseline = None
else:
baseline = Image.open(os.path.join(mask_scrape_dir, mask_scrapes[0]))
-
+
mask_scrape_filename = os.path.join(mask_scrape_dir,
time.strftime("%y%m%d-%H%M%S.bmp"))
-
+
# Do the scrape
result = scraper.Scrape(
[url.url], mask_scrape_dir, size, (0, 0),
command["--timeout"], path=command["--browserpath"],
filename=mask_scrape_filename)
-
+
if result:
# Return value other than None means an error
print " Scrape failed with error '%r'" % result
@@ -189,16 +189,16 @@ def ExecuteMaskmaker(command):
if url.errors >= errors:
print " ** Exceeded maximum error count for this URL, giving up"
continue
-
+
# Load the new scrape
scrape = Image.open(mask_scrape_filename)
-
+
# Calculate the difference between the new scrape and the baseline,
# subject to the current mask
if baseline:
diff = ImageChops.multiply(ImageChops.difference(scrape, baseline),
mask.convert(scrape.mode))
-
+
# If the difference is none, there's nothing to update
if max(diff.getextrema()) == (0, 0):
print " Scrape identical to baseline, no change in mask"
@@ -221,10 +221,10 @@ def ExecuteMaskmaker(command):
# a monochrome bitmap. If the original RGB image were converted
# directly to monochrome, PIL would dither it.
diff = diff.convert("L").point([255]+[0]*255, "1")
-
+
# count the number of different pixels
diff_pixels = diff.getcolors()[0][0]
-
+
# is this too much?
diff_pixel_percent = diff_pixels * 100.0 / (mask.size[0]*mask.size[1])
if diff_pixel_percent > command["--threshhold"]:
@@ -234,10 +234,10 @@ def ExecuteMaskmaker(command):
print " Scrape differed in %d pixels, updating mask" % diff_pixels
mask = ImageChops.multiply(mask, diff)
mask.save(mask_filename)
-
+
# reset the number of consecutive "good" scrapes
url.consecutive_successes = 0
-
+
# Remove URLs whose mask is deemed done
complete_list.extend(
[url for url in url_list if url.consecutive_successes >= scrapes])
@@ -247,16 +247,16 @@ def ExecuteMaskmaker(command):
url for url in url_list if
url.consecutive_successes < scrapes and
url.errors < errors]
-
+
scrape_pass += 1
print "**Done with scrape pass %d\n" % scrape_pass
-
+
if scrape_pass >= command["--giveup"]:
print "**Exceeded giveup threshhold. Giving up."
else:
print "Waiting %d seconds..." % command["--wait"]
time.sleep(command["--wait"])
-
+
print
print "*** MASKMAKER COMPLETE ***"
print "Summary report:"
diff --git a/tools/site_compare/commands/measure.py b/tools/site_compare/commands/measure.py
index 1815a3d2..086fcbe 100644
--- a/tools/site_compare/commands/measure.py
+++ b/tools/site_compare/commands/measure.py
@@ -40,14 +40,14 @@ def CreateCommand(cmdline):
def ExecuteMeasure(command):
"""Executes the Measure command."""
-
+
def LogResult(url, proc, wnd, result):
"""Write the result of the browse to the log file."""
log_file.write(result)
log_file = open(command["--logfile"], "w")
- browser_iterate.Iterate(command, LogResult)
+ browser_iterate.Iterate(command, LogResult)
# Close the log file and return. We're done.
log_file.close()
diff --git a/tools/site_compare/commands/scrape.py b/tools/site_compare/commands/scrape.py
index 21a00ce..1c47cab 100644
--- a/tools/site_compare/commands/scrape.py
+++ b/tools/site_compare/commands/scrape.py
@@ -41,7 +41,7 @@ def CreateCommand(cmdline):
def ExecuteScrape(command):
"""Executes the Scrape command."""
-
+
def ScrapeResult(url, proc, wnd, result):
"""Capture and save the scrape."""
if log_file: log_file.write(result)
@@ -49,12 +49,12 @@ def ExecuteScrape(command):
# Scrape the page
image = windowing.ScrapeWindow(wnd)
filename = windowing.URLtoFilename(url, command["--outdir"], ".bmp")
- image.save(filename)
+ image.save(filename)
if command["--logfile"]: log_file = open(command["--logfile"], "w")
else: log_file = None
- browser_iterate.Iterate(command, ScrapeResult)
+ browser_iterate.Iterate(command, ScrapeResult)
# Close the log file and return. We're done.
if log_file: log_file.close()
diff --git a/tools/site_compare/commands/timeload.py b/tools/site_compare/commands/timeload.py
index 554d3b6..ca5b0db 100644
--- a/tools/site_compare/commands/timeload.py
+++ b/tools/site_compare/commands/timeload.py
@@ -6,7 +6,7 @@
"""SiteCompare command to time page loads
Loads a series of URLs in a series of browsers (and browser versions)
-and measures how long the page takes to load in each. Outputs a
+and measures how long the page takes to load in each. Outputs a
comma-delimited file. The first line is "URL,[browser names", each
additional line is a URL follored by comma-delimited times (in seconds),
or the string "timeout" or "crashed".
@@ -67,44 +67,44 @@ def CreateCommand(cmdline):
cmd.AddArgument(
["-sz", "--size"], "Browser window size", default=(800, 600), type="coords")
-
+
def ExecuteTimeLoad(command):
"""Executes the TimeLoad command."""
browsers = command["--browsers"].split(",")
num_browsers = len(browsers)
-
+
if command["--browserversions"]:
browser_versions = command["--browserversions"].split(",")
else:
browser_versions = [None] * num_browsers
-
+
if command["--browserpaths"]:
browser_paths = command["--browserpaths"].split(",")
else:
browser_paths = [None] * num_browsers
-
+
if len(browser_versions) != num_browsers:
raise ValueError(
"--browserversions must be same length as --browser_paths")
if len(browser_paths) != num_browsers:
raise ValueError(
"--browserversions must be same length as --browser_paths")
-
+
if [b for b in browsers if b not in ["chrome", "ie", "firefox"]]:
raise ValueError("unknown browsers: %r" % b)
-
+
scraper_list = []
-
+
for b in xrange(num_browsers):
version = browser_versions[b]
if not version: version = None
-
+
scraper = scrapers.GetScraper( (browsers[b], version) )
if not scraper:
- raise ValueError("could not find scraper for (%r, %r)" %
+ raise ValueError("could not find scraper for (%r, %r)" %
(browsers[b], version))
scraper_list.append(scraper)
-
+
if command["--url"]:
url_list = [command["--url"]]
else:
@@ -115,32 +115,32 @@ def ExecuteTimeLoad(command):
endline = command["--endline"]
url_list = [url.strip() for url in
open(command["--list"], "r").readlines()[startline:endline]]
-
+
log_file = open(command["--logfile"], "w")
-
+
log_file.write("URL")
for b in xrange(num_browsers):
log_file.write(",%s" % browsers[b])
-
+
if browser_versions[b]: log_file.write(" %s" % browser_versions[b])
log_file.write("\n")
-
+
results = {}
for url in url_list:
results[url] = [None] * num_browsers
-
+
for b in xrange(num_browsers):
result = scraper_list[b].Time(url_list, command["--size"],
command["--timeout"],
path=browser_paths[b])
-
+
for (url, time) in result:
results[url][b] = time
-
+
# output the results
for url in url_list:
log_file.write(url)
for b in xrange(num_browsers):
log_file.write(",%r" % results[url][b])
-
+
diff --git a/tools/site_compare/drivers/__init__.py b/tools/site_compare/drivers/__init__.py
index befc1353..fa9f1c2 100644
--- a/tools/site_compare/drivers/__init__.py
+++ b/tools/site_compare/drivers/__init__.py
@@ -9,7 +9,7 @@ __author__ = 'jhaas@google.com (Jonathan Haas)'
import sys
platform_dir = sys.platform
-
+
keyboard = __import__(platform_dir+".keyboard", globals(), locals(), [''])
mouse = __import__(platform_dir+".mouse", globals(), locals(), [''])
windowing = __import__(platform_dir+".windowing", globals(), locals(), [''])
diff --git a/tools/site_compare/drivers/win32/keyboard.py b/tools/site_compare/drivers/win32/keyboard.py
index a25df5e..246e14c 100644
--- a/tools/site_compare/drivers/win32/keyboard.py
+++ b/tools/site_compare/drivers/win32/keyboard.py
@@ -25,45 +25,45 @@ import win32con # Windows constants
def PressKey(down, key):
"""Presses or unpresses a key.
-
+
Uses keybd_event to simulate either depressing or releasing
a key
-
+
Args:
down: Whether the key is to be pressed or released
key: Virtual key code of key to press or release
"""
-
- # keybd_event injects key events at a very low level (it's the
+
+ # keybd_event injects key events at a very low level (it's the
# Windows API keyboard device drivers call) so this is a very
# reliable way of simulating user input
win32api.keybd_event(key, 0, (not down) * win32con.KEYEVENTF_KEYUP)
-
-
+
+
def TypeKey(key, keystroke_time=0):
"""Simulate a keypress of a virtual key.
-
+
Args:
key: which key to press
keystroke_time: length of time (in seconds) to "hold down" the key
Note that zero works just fine
-
+
Returns:
None
"""
-
+
# This just wraps a pair of PressKey calls with an intervening delay
PressKey(True, key)
time.sleep(keystroke_time)
PressKey(False, key)
-
+
def TypeString(string_to_type,
use_modifiers=False,
keystroke_time=0,
time_between_keystrokes=0):
"""Simulate typing a string on the keyboard.
-
+
Args:
string_to_type: the string to print
use_modifiers: specifies whether the following modifier characters
@@ -79,27 +79,27 @@ def TypeString(string_to_type,
nonprintable keys (F-keys, ESC, arrow keys, etc),
support for explicit control of left vs. right ALT or SHIFT,
support for Windows key
-
+
keystroke_time: length of time (in secondes) to "hold down" the key
time_between_keystrokes: length of time (seconds) to pause between keys
-
+
Returns:
None
"""
-
+
shift_held = win32api.GetAsyncKeyState(win32con.VK_SHIFT ) < 0
ctrl_held = win32api.GetAsyncKeyState(win32con.VK_CONTROL) < 0
alt_held = win32api.GetAsyncKeyState(win32con.VK_MENU ) < 0
-
+
next_escaped = False
escape_chars = {
'a': '\a', 'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t', 'v': '\v'}
-
+
for char in string_to_type:
vk = None
handled = False
-
- # Check to see if this is the start or end of a modified block (that is,
+
+ # Check to see if this is the start or end of a modified block (that is,
# {abc} for ALT-modified keys or [abc] for CTRL-modified keys
if use_modifiers and not next_escaped:
handled = True
@@ -117,17 +117,17 @@ def TypeString(string_to_type,
PressKey(False, win32con.VK_CONTROL)
else:
handled = False
-
+
# If this is an explicitly-escaped character, replace it with the
# appropriate code
if next_escaped and char in escape_chars: char = escape_chars[char]
-
+
# If this is \p, pause for one second.
if next_escaped and char == 'p':
time.sleep(1)
next_escaped = False
handled = True
-
+
# If this is \(d), press F key
if next_escaped and char.isdigit():
fkey = int(char)
@@ -139,28 +139,28 @@ def TypeString(string_to_type,
if not next_escaped and char == "\\":
next_escaped = True
handled = True
-
+
# If we make it here, it's not a special character, or it's an
# escaped special character which should be treated as a literal
if not handled:
next_escaped = False
if not vk: vk = win32api.VkKeyScan(char)
-
+
# VkKeyScan() returns the scan code in the low byte. The upper
# byte specifies modifiers necessary to produce the given character
# from the given scan code. The only one we're concerned with at the
- # moment is Shift. Determine the shift state and compare it to the
+ # moment is Shift. Determine the shift state and compare it to the
# current state... if it differs, press or release the shift key.
new_shift_held = bool(vk & (1<<8))
-
+
if new_shift_held != shift_held:
PressKey(new_shift_held, win32con.VK_SHIFT)
shift_held = new_shift_held
-
+
# Type the key with the specified length, then wait the specified delay
TypeKey(vk & 0xFF, keystroke_time)
time.sleep(time_between_keystrokes)
-
+
# Release the modifier keys, if held
if shift_held: PressKey(False, win32con.VK_SHIFT)
if ctrl_held: PressKey(False, win32con.VK_CONTROL)
@@ -168,18 +168,18 @@ def TypeString(string_to_type,
if __name__ == "__main__":
# We're being invoked rather than imported. Let's do some tests
-
+
# Press command-R to bring up the Run dialog
PressKey(True, win32con.VK_LWIN)
TypeKey(ord('R'))
PressKey(False, win32con.VK_LWIN)
-
+
# Wait a sec to make sure it comes up
time.sleep(1)
-
+
# Invoke Notepad through the Run dialog
TypeString("wordpad\n")
-
+
# Wait another sec, then start typing
time.sleep(1)
TypeString("This is a test of SiteCompare's Keyboard.py module.\n\n")
@@ -194,5 +194,5 @@ if __name__ == "__main__":
use_modifiers=True,
keystroke_time=0.05,
time_between_keystrokes=0.05)
-
-
+
+
diff --git a/tools/site_compare/drivers/win32/mouse.py b/tools/site_compare/drivers/win32/mouse.py
index afdb2ea..bd16272 100644
--- a/tools/site_compare/drivers/win32/mouse.py
+++ b/tools/site_compare/drivers/win32/mouse.py
@@ -19,25 +19,25 @@ import win32gui # for window functions
def ScreenToMouse(pt):
"""Convert a value in screen coordinates to mouse coordinates.
-
+
Mouse coordinates are specified as a percentage of screen dimensions,
normalized to 16 bits. 0 represents the far left/top of the screen,
65535 represents the far right/bottom. This function assumes that
the size of the screen is fixed at module load time and does not change
-
+
Args:
pt: the point of the coords to convert
-
+
Returns:
the converted point
"""
-
+
# Initialize the screen dimensions on first execution. Note that this
# function assumes that the screen dimensions do not change during run.
if not ScreenToMouse._SCREEN_DIMENSIONS:
desktop = win32gui.GetClientRect(win32gui.GetDesktopWindow())
ScreenToMouse._SCREEN_DIMENSIONS = (desktop[2], desktop[3])
-
+
return ((65535 * pt[0]) / ScreenToMouse._SCREEN_DIMENSIONS[0],
(65535 * pt[1]) / ScreenToMouse._SCREEN_DIMENSIONS[1])
@@ -46,11 +46,11 @@ ScreenToMouse._SCREEN_DIMENSIONS = None
def PressButton(down, button='left'):
"""Simulate a mouse button press or release at the current mouse location.
-
+
Args:
down: whether the button is pressed or released
button: which button is pressed
-
+
Returns:
None
"""
@@ -61,127 +61,127 @@ def PressButton(down, button='left'):
'middle': (win32con.MOUSEEVENTF_MIDDLEUP, win32con.MOUSEEVENTF_MIDDLEDOWN),
'right': (win32con.MOUSEEVENTF_RIGHTUP, win32con.MOUSEEVENTF_RIGHTDOWN)
}
-
+
# hit the button
win32api.mouse_event(flags[button][down], 0, 0)
-
+
def ClickButton(button='left', click_time=0):
"""Press and release a mouse button at the current mouse location.
-
+
Args:
button: which button to click
click_time: duration between press and release
-
+
Returns:
None
"""
PressButton(True, button)
time.sleep(click_time)
PressButton(False, button)
-
-
+
+
def DoubleClickButton(button='left', click_time=0, time_between_clicks=0):
"""Double-click a mouse button at the current mouse location.
-
+
Args:
button: which button to click
click_time: duration between press and release
time_between_clicks: time to pause between clicks
-
+
Returns:
None
"""
ClickButton(button, click_time)
time.sleep(time_between_clicks)
ClickButton(button, click_time)
-
+
def MoveToLocation(pos, duration=0, tick=0.01):
"""Move the mouse cursor to a specified location, taking the specified time.
-
+
Args:
pos: position (in screen coordinates) to move to
duration: amount of time the move should take
tick: amount of time between successive moves of the mouse
-
+
Returns:
None
"""
# calculate the number of moves to reach the destination
num_steps = (duration/tick)+1
-
+
# get the current and final mouse position in mouse coords
current_location = ScreenToMouse(win32gui.GetCursorPos())
end_location = ScreenToMouse(pos)
-
+
# Calculate the step size
step_size = ((end_location[0]-current_location[0])/num_steps,
(end_location[1]-current_location[1])/num_steps)
step = 0
-
+
while step < num_steps:
# Move the mouse one step
current_location = (current_location[0]+step_size[0],
current_location[1]+step_size[1])
-
+
# Coerce the coords to int to avoid a warning from pywin32
win32api.mouse_event(
win32con.MOUSEEVENTF_MOVE|win32con.MOUSEEVENTF_ABSOLUTE,
int(current_location[0]), int(current_location[1]))
-
+
step += 1
time.sleep(tick)
-
-
+
+
def ClickAtLocation(pos, button='left', click_time=0):
"""Simulate a mouse click in a particular location, in screen coordinates.
-
+
Args:
pos: position in screen coordinates (x,y)
button: which button to click
click_time: duration of the click
-
+
Returns:
None
"""
MoveToLocation(pos)
ClickButton(button, click_time)
-
+
def ClickInWindow(hwnd, offset=None, button='left', click_time=0):
"""Simulate a user mouse click in the center of a window.
-
+
Args:
hwnd: handle of the window to click in
offset: where to click, defaults to dead center
button: which button to click
click_time: duration of the click
-
+
Returns:
Nothing
"""
-
+
rect = win32gui.GetClientRect(hwnd)
if offset is None: offset = (rect[2]/2, rect[3]/2)
# get the screen coordinates of the window's center
pos = win32gui.ClientToScreen(hwnd, offset)
-
+
ClickAtLocation(pos, button, click_time)
-
+
def DoubleClickInWindow(
hwnd, offset=None, button='left', click_time=0, time_between_clicks=0.1):
"""Simulate a user mouse double click in the center of a window.
-
+
Args:
hwnd: handle of the window to click in
offset: where to click, defaults to dead center
button: which button to click
click_time: duration of the clicks
time_between_clicks: length of time to pause between clicks
-
+
Returns:
Nothing
"""
@@ -191,13 +191,13 @@ def DoubleClickInWindow(
if __name__ == "__main__":
# We're being invoked rather than imported. Let's do some tests
-
+
screen_size = win32gui.GetClientRect(win32gui.GetDesktopWindow())
screen_size = (screen_size[2], screen_size[3])
-
+
# move the mouse (instantly) to the upper right corner
MoveToLocation((screen_size[0], 0))
-
+
# move the mouse (over five seconds) to the lower left corner
MoveToLocation((0, screen_size[1]), 5)
@@ -209,10 +209,10 @@ if __name__ == "__main__":
# wait a bit, then click the right button to open the context menu
time.sleep(3)
ClickButton('right')
-
+
# move the mouse away and then click the left button to dismiss the
# context menu
MoveToLocation((screen_size[0]/2, screen_size[1]/2), 3)
MoveToLocation((0, 0), 3)
ClickButton()
-
+
diff --git a/tools/site_compare/drivers/win32/windowing.py b/tools/site_compare/drivers/win32/windowing.py
index 5bc37f8..fe77c56 100644
--- a/tools/site_compare/drivers/win32/windowing.py
+++ b/tools/site_compare/drivers/win32/windowing.py
@@ -24,7 +24,7 @@ import win32process
def FindChildWindows(hwnd, path):
"""Find a set of windows through a path specification.
-
+
Args:
hwnd: Handle of the parent window
path: Path to the window to find. Has the following form:
@@ -32,12 +32,12 @@ def FindChildWindows(hwnd, path):
The slashes specify the "path" to the child window.
The text is the window class, a pipe (if present) is a title.
* is a wildcard and will find all child windows at that level
-
+
Returns:
A list of the windows that were found
"""
windows_to_check = [hwnd]
-
+
# The strategy will be to take windows_to_check and use it
# to find a list of windows that match the next specification
# in the path, then repeat with the list of found windows as the
@@ -45,7 +45,7 @@ def FindChildWindows(hwnd, path):
for segment in path.split("/"):
windows_found = []
check_values = segment.split("|")
-
+
# check_values is now a list with the first element being
# the window class, the second being the window caption.
# If the class is absent (or wildcarded) set it to None
@@ -53,7 +53,7 @@ def FindChildWindows(hwnd, path):
# If the window caption is also absent, force it to None as well
if len(check_values) == 1: check_values.append(None)
-
+
# Loop through the list of windows to check
for window_check in windows_to_check:
window_found = None
@@ -70,26 +70,26 @@ def FindChildWindows(hwnd, path):
window_found = 0
else:
raise e
-
+
# If FindWindowEx struck gold, add to our list of windows found
if window_found: windows_found.append(window_found)
-
+
# The windows we found become the windows to check for the next segment
windows_to_check = windows_found
-
+
return windows_found
def FindChildWindow(hwnd, path):
"""Find a window through a path specification.
-
+
This method is a simple wrapper for FindChildWindows() for the
case (the majority case) where you expect to find a single window
-
+
Args:
hwnd: Handle of the parent window
path: Path to the window to find. See FindChildWindows()
-
+
Returns:
The window that was found
"""
@@ -98,36 +98,36 @@ def FindChildWindow(hwnd, path):
def ScrapeWindow(hwnd, rect=None):
"""Scrape a visible window and return its contents as a bitmap.
-
+
Args:
hwnd: handle of the window to scrape
rect: rectangle to scrape in client coords, defaults to the whole thing
If specified, it's a 4-tuple of (left, top, right, bottom)
-
+
Returns:
An Image containing the scraped data
"""
# Activate the window
SetForegroundWindow(hwnd)
-
+
# If no rectangle was specified, use the fill client rectangle
if not rect: rect = win32gui.GetClientRect(hwnd)
-
+
upper_left = win32gui.ClientToScreen(hwnd, (rect[0], rect[1]))
lower_right = win32gui.ClientToScreen(hwnd, (rect[2], rect[3]))
rect = upper_left+lower_right
-
+
return PIL.ImageGrab.grab(rect)
-
+
def SetForegroundWindow(hwnd):
"""Bring a window to the foreground."""
win32gui.SetForegroundWindow(hwnd)
-
-
+
+
def InvokeAndWait(path, cmdline="", timeout=10, tick=1.):
"""Invoke an application and wait for it to bring up a window.
-
+
Args:
path: full path to the executable to invoke
cmdline: command line to pass to executable
@@ -138,7 +138,7 @@ def InvokeAndWait(path, cmdline="", timeout=10, tick=1.):
A tuple of handles to the process and the application's window,
or (None, None) if it timed out waiting for the process
"""
-
+
def EnumWindowProc(hwnd, ret):
"""Internal enumeration func, checks for visibility and proper PID."""
if win32gui.IsWindowVisible(hwnd): # don't bother even checking hidden wnds
@@ -147,12 +147,12 @@ def InvokeAndWait(path, cmdline="", timeout=10, tick=1.):
ret[1] = hwnd
return 0 # 0 means stop enumeration
return 1 # 1 means continue enumeration
-
+
# We don't need to change anything about the startupinfo structure
# (the default is quite sufficient) but we need to create it just the
# same.
sinfo = win32process.STARTUPINFO()
-
+
proc = win32process.CreateProcess(
path, # path to new process's executable
cmdline, # application's command line
@@ -168,16 +168,16 @@ def InvokeAndWait(path, cmdline="", timeout=10, tick=1.):
# some point we may care about the other members, but for now, all
# we're after is the pid
pid = proc[2]
-
+
# Enumeration APIs can take an arbitrary integer, usually a pointer,
# to be passed to the enumeration function. We'll pass a pointer to
# a structure containing the PID we're looking for, and an empty out
# parameter to hold the found window ID
ret = [pid, None]
-
+
tries_until_timeout = timeout/tick
num_tries = 0
-
+
# Enumerate top-level windows, look for one with our PID
while num_tries < tries_until_timeout and ret[1] is None:
try:
@@ -186,7 +186,7 @@ def InvokeAndWait(path, cmdline="", timeout=10, tick=1.):
# error 0 isn't an error, it just meant the enumeration was
# terminated early
if e[0]: raise e
-
+
time.sleep(tick)
num_tries += 1
@@ -197,11 +197,11 @@ def InvokeAndWait(path, cmdline="", timeout=10, tick=1.):
def WaitForProcessExit(proc, timeout=None):
"""Waits for a given process to terminate.
-
+
Args:
proc: handle to process
timeout: timeout (in seconds). None = wait indefinitely
-
+
Returns:
True if process ended, False if timed out
"""
@@ -210,26 +210,26 @@ def WaitForProcessExit(proc, timeout=None):
else:
# convert sec to msec
timeout *= 1000
-
+
return (win32event.WaitForSingleObject(proc, timeout) ==
win32event.WAIT_OBJECT_0)
def WaitForThrobber(hwnd, rect=None, timeout=20, tick=0.1, done=10):
"""Wait for a browser's "throbber" (loading animation) to complete.
-
+
Args:
hwnd: window containing the throbber
rect: rectangle of the throbber, in client coords. If None, whole window
timeout: if the throbber is still throbbing after this long, give up
tick: how often to check the throbber
done: how long the throbber must be unmoving to be considered done
-
+
Returns:
Number of seconds waited, -1 if timed out
"""
if not rect: rect = win32gui.GetClientRect(hwnd)
-
+
# last_throbber will hold the results of the preceding scrape;
# we'll compare it against the current scrape to see if we're throbbing
last_throbber = ScrapeWindow(hwnd, rect)
@@ -239,7 +239,7 @@ def WaitForThrobber(hwnd, rect=None, timeout=20, tick=0.1, done=10):
while time.clock() < timeout_clock:
time.sleep(tick)
-
+
current_throbber = ScrapeWindow(hwnd, rect)
if current_throbber.tostring() != last_throbber.tostring():
last_throbber = current_throbber
@@ -247,27 +247,27 @@ def WaitForThrobber(hwnd, rect=None, timeout=20, tick=0.1, done=10):
else:
if time.clock() - last_changed_clock > done:
return last_changed_clock - start_clock
-
+
return -1
def MoveAndSizeWindow(wnd, position=None, size=None, child=None):
"""Moves and/or resizes a window.
-
+
Repositions and resizes a window. If a child window is provided,
the parent window is resized so the child window has the given size
-
+
Args:
wnd: handle of the frame window
position: new location for the frame window
size: new size for the frame window (or the child window)
child: handle of the child window
-
+
Returns:
None
"""
rect = win32gui.GetWindowRect(wnd)
-
+
if position is None: position = (rect[0], rect[1])
if size is None:
size = (rect[2]-rect[0], rect[3]-rect[1])
@@ -276,7 +276,7 @@ def MoveAndSizeWindow(wnd, position=None, size=None, child=None):
slop = (rect[2]-rect[0]-child_rect[2]+child_rect[0],
rect[3]-rect[1]-child_rect[3]+child_rect[1])
size = (size[0]+slop[0], size[1]+slop[1])
-
+
win32gui.MoveWindow(wnd, # window to move
position[0], # new x coord
position[1], # new y coord
@@ -287,46 +287,46 @@ def MoveAndSizeWindow(wnd, position=None, size=None, child=None):
def EndProcess(proc, code=0):
"""Ends a process.
-
+
Wraps the OS TerminateProcess call for platform-independence
-
+
Args:
proc: process ID
code: process exit code
-
+
Returns:
None
"""
win32process.TerminateProcess(proc, code)
-
-
+
+
def URLtoFilename(url, path=None, extension=None):
"""Converts a URL to a filename, given a path.
-
+
This in theory could cause collisions if two URLs differ only
in unprintable characters (eg. http://www.foo.com/?bar and
http://www.foo.com/:bar. In practice this shouldn't be a problem.
-
+
Args:
url: The URL to convert
path: path to the directory to store the file
extension: string to append to filename
-
+
Returns:
filename
"""
trans = string.maketrans(r'\/:*?"<>|', '_________')
-
+
if path is None: path = ""
if extension is None: extension = ""
if len(path) > 0 and path[-1] != '\\': path += '\\'
url = url.translate(trans)
return "%s%s%s" % (path, url, extension)
-
+
def PreparePath(path):
"""Ensures that a given path exists, making subdirectories if necessary.
-
+
Args:
path: fully-qualified path of directory to ensure exists
@@ -341,11 +341,11 @@ def PreparePath(path):
if __name__ == "__main__":
PreparePath(r"c:\sitecompare\scrapes\ie7")
# We're being invoked rather than imported. Let's do some tests
-
+
# Hardcode IE's location for the purpose of this test
(proc, wnd) = InvokeAndWait(
r"c:\program files\internet explorer\iexplore.exe")
-
+
# Find the browser pane in the IE window
browser = FindChildWindow(
wnd, "TabWindowClass/Shell DocObject View/Internet Explorer_Server")
@@ -355,8 +355,8 @@ if __name__ == "__main__":
# Take a screenshot
i = ScrapeWindow(browser)
-
+
i.show()
-
+
EndProcess(proc, 0)
diff --git a/tools/site_compare/operators/__init__.py b/tools/site_compare/operators/__init__.py
index 02eac07..f60e8e8 100644
--- a/tools/site_compare/operators/__init__.py
+++ b/tools/site_compare/operators/__init__.py
@@ -9,18 +9,18 @@ __author__ = 'jhaas@google.com (Jonathan Haas)'
def GetOperator(operator):
"""Given an operator by name, returns its module.
-
+
Args:
operator: string describing the comparison
-
+
Returns:
module
"""
-
+
# TODO(jhaas): come up with a happy way of integrating multiple operators
# with different, possibly divergent and possibly convergent, operators.
-
+
module = __import__(operator, globals(), locals(), [''])
-
+
return module
diff --git a/tools/site_compare/operators/equals.py b/tools/site_compare/operators/equals.py
index c7654e9..4054fa6 100644
--- a/tools/site_compare/operators/equals.py
+++ b/tools/site_compare/operators/equals.py
@@ -11,31 +11,31 @@ from PIL import ImageChops
def Compare(file1, file2, **kwargs):
"""Compares two images to see if they're identical.
-
+
Args:
file1: path to first image to compare
file2: path to second image to compare
kwargs: unused for this operator
-
+
Returns:
None if the images are identical
A tuple of (errorstring, image) if they're not
"""
kwargs = kwargs # unused parameter
-
+
im1 = Image.open(file1)
im2 = Image.open(file2)
-
+
if im1.size != im2.size:
return ("The images are of different size (%s vs %s)" %
(im1.size, im2.size), im1)
diff = ImageChops.difference(im1, im2)
-
+
if max(diff.getextrema()) != (0, 0):
return ("The images differ", diff)
else:
return None
-
-
-
+
+
+
diff --git a/tools/site_compare/operators/equals_with_mask.py b/tools/site_compare/operators/equals_with_mask.py
index fd4000b..d6abd53 100644
--- a/tools/site_compare/operators/equals_with_mask.py
+++ b/tools/site_compare/operators/equals_with_mask.py
@@ -13,49 +13,49 @@ import os.path
def Compare(file1, file2, **kwargs):
"""Compares two images to see if they're identical subject to a mask.
-
+
An optional directory containing masks is supplied. If a mask exists
which matches file1's name, areas under the mask where it's black
are ignored.
-
+
Args:
file1: path to first image to compare
file2: path to second image to compare
kwargs: ["maskdir"] contains the directory holding the masks
-
+
Returns:
None if the images are identical
A tuple of (errorstring, image) if they're not
"""
-
+
maskdir = None
if "maskdir" in kwargs:
maskdir = kwargs["maskdir"]
-
+
im1 = Image.open(file1)
im2 = Image.open(file2)
-
+
if im1.size != im2.size:
return ("The images are of different size (%r vs %r)" %
(im1.size, im2.size), im1)
diff = ImageChops.difference(im1, im2)
-
+
if maskdir:
maskfile = os.path.join(maskdir, os.path.basename(file1))
if os.path.exists(maskfile):
mask = Image.open(maskfile)
-
+
if mask.size != im1.size:
return ("The mask is of a different size than the images (%r vs %r)" %
(mask.size, im1.size), mask)
-
+
diff = ImageChops.multiply(diff, mask.convert(diff.mode))
-
+
if max(diff.getextrema()) != (0, 0):
return ("The images differ", diff)
else:
return None
-
-
-
+
+
+
diff --git a/tools/site_compare/scrapers/__init__.py b/tools/site_compare/scrapers/__init__.py
index 08790aa..cb82b2b 100644
--- a/tools/site_compare/scrapers/__init__.py
+++ b/tools/site_compare/scrapers/__init__.py
@@ -12,23 +12,23 @@ import types
def GetScraper(browser):
"""Given a browser and an optional version, returns the scraper module.
-
+
Args:
browser: either a string (browser name) or a tuple (name, version)
-
+
Returns:
module
"""
-
+
if type(browser) == types.StringType: browser = (browser, None)
-
+
package = __import__(browser[0], globals(), locals(), [''])
module = package.GetScraper(browser[1])
if browser[1] is not None: module.version = browser[1]
-
+
return module
# if invoked rather than imported, do some tests
if __name__ == "__main__":
print GetScraper("IE")
- \ No newline at end of file
+
diff --git a/tools/site_compare/scrapers/chrome/__init__.py b/tools/site_compare/scrapers/chrome/__init__.py
index 2ba76c4..6342525 100644
--- a/tools/site_compare/scrapers/chrome/__init__.py
+++ b/tools/site_compare/scrapers/chrome/__init__.py
@@ -5,19 +5,19 @@
"""Selects the appropriate scraper for Chrome."""
__author__ = 'jhaas@google.com (Jonathan Haas)'
-
+
def GetScraper(version):
"""Returns the scraper module for the given version.
-
+
Args:
version: version string of Chrome, or None for most recent
-
+
Returns:
scrape module for given version
"""
if version is None:
version = "0.1.101.0"
-
+
parsed_version = [int(x) for x in version.split(".")]
if (parsed_version[0] > 0 or
@@ -29,10 +29,10 @@ def GetScraper(version):
scraper_version = "chrome01970"
return __import__(scraper_version, globals(), locals(), [''])
-
+
# if invoked rather than imported, test
if __name__ == "__main__":
version = "0.1.101.0"
-
+
print GetScraper(version).version
- \ No newline at end of file
+
diff --git a/tools/site_compare/scrapers/chrome/chrome011010.py b/tools/site_compare/scrapers/chrome/chrome011010.py
index 3d62d07..b4f816f 100644
--- a/tools/site_compare/scrapers/chrome/chrome011010.py
+++ b/tools/site_compare/scrapers/chrome/chrome011010.py
@@ -19,7 +19,7 @@ def GetChromeRenderPane(wnd):
def Scrape(urls, outdir, size, pos, timeout=20, **kwargs):
"""Invoke a browser, send it to a series of URLs, and save its output.
-
+
Args:
urls: list of URLs to scrape
outdir: directory to place output
@@ -27,18 +27,18 @@ def Scrape(urls, outdir, size, pos, timeout=20, **kwargs):
pos: position of browser window
timeout: amount of time to wait for page to load
kwargs: miscellaneous keyword args
-
+
Returns:
None if succeeded, else an error code
"""
chromebase.GetChromeRenderPane = GetChromeRenderPane
-
+
return chromebase.Scrape(urls, outdir, size, pos, timeout, kwargs)
def Time(urls, size, timeout, **kwargs):
"""Forwards the Time command to chromebase."""
chromebase.GetChromeRenderPane = GetChromeRenderPane
-
+
return chromebase.Time(urls, size, timeout, kwargs)
diff --git a/tools/site_compare/scrapers/chrome/chrome01970.py b/tools/site_compare/scrapers/chrome/chrome01970.py
index c1ef79f..54bc670 100644
--- a/tools/site_compare/scrapers/chrome/chrome01970.py
+++ b/tools/site_compare/scrapers/chrome/chrome01970.py
@@ -19,7 +19,7 @@ def GetChromeRenderPane(wnd):
def Scrape(urls, outdir, size, pos, timeout=20, **kwargs):
"""Invoke a browser, send it to a series of URLs, and save its output.
-
+
Args:
urls: list of URLs to scrape
outdir: directory to place output
@@ -27,18 +27,18 @@ def Scrape(urls, outdir, size, pos, timeout=20, **kwargs):
pos: position of browser window
timeout: amount of time to wait for page to load
kwargs: miscellaneous keyword args
-
+
Returns:
None if succeeded, else an error code
"""
chromebase.GetChromeRenderPane = GetChromeRenderPane
-
+
return chromebase.Scrape(urls, outdir, size, pos, timeout, kwargs)
-
+
def Time(urls, size, timeout, **kwargs):
"""Forwards the Time command to chromebase."""
chromebase.GetChromeRenderPane = GetChromeRenderPane
-
+
return chromebase.Time(urls, size, timeout, kwargs)
-
+
diff --git a/tools/site_compare/scrapers/chrome/chromebase.py b/tools/site_compare/scrapers/chrome/chromebase.py
index 085f376..aba17c1 100644
--- a/tools/site_compare/scrapers/chrome/chromebase.py
+++ b/tools/site_compare/scrapers/chrome/chromebase.py
@@ -18,17 +18,17 @@ DEFAULT_PATH = r"k:\chrome.exe"
def InvokeBrowser(path):
"""Invoke the Chrome browser.
-
+
Args:
path: full path to browser
-
+
Returns:
A tuple of (main window, process handle, address bar, render pane)
"""
-
+
# Reuse an existing instance of the browser if we can find one. This
# may not work correctly, especially if the window is behind other windows.
-
+
# TODO(jhaas): make this work with Vista
wnds = windowing.FindChildWindows(0, "Chrome_XPFrame")
if len(wnds):
@@ -37,17 +37,17 @@ def InvokeBrowser(path):
else:
# Invoke Chrome
(proc, wnd) = windowing.InvokeAndWait(path)
-
+
# Get windows we'll need
address_bar = windowing.FindChildWindow(wnd, "Chrome_AutocompleteEdit")
render_pane = GetChromeRenderPane(wnd)
-
+
return (wnd, proc, address_bar, render_pane)
-
+
def Scrape(urls, outdir, size, pos, timeout, kwargs):
"""Invoke a browser, send it to a series of URLs, and save its output.
-
+
Args:
urls: list of URLs to scrape
outdir: directory to place output
@@ -55,39 +55,39 @@ def Scrape(urls, outdir, size, pos, timeout, kwargs):
pos: position of browser window
timeout: amount of time to wait for page to load
kwargs: miscellaneous keyword args
-
+
Returns:
None if success, else an error string
"""
if "path" in kwargs and kwargs["path"]: path = kwargs["path"]
else: path = DEFAULT_PATH
-
+
(wnd, proc, address_bar, render_pane) = InvokeBrowser(path)
-
+
# Resize and reposition the frame
windowing.MoveAndSizeWindow(wnd, pos, size, render_pane)
-
+
# Visit each URL we're given
if type(urls) in types.StringTypes: urls = [urls]
timedout = False
-
+
for url in urls:
# Double-click in the address bar, type the name, and press Enter
mouse.ClickInWindow(address_bar)
keyboard.TypeString(url, 0.1)
keyboard.TypeString("\n")
-
+
# Wait for the page to finish loading
load_time = windowing.WaitForThrobber(wnd, (20, 16, 36, 32), timeout)
timedout = load_time < 0
-
+
if timedout:
break
-
+
# Scrape the page
image = windowing.ScrapeWindow(render_pane)
-
+
# Save to disk
if "filename" in kwargs:
if callable(kwargs["filename"]):
@@ -97,68 +97,68 @@ def Scrape(urls, outdir, size, pos, timeout, kwargs):
else:
filename = windowing.URLtoFilename(url, outdir, ".bmp")
image.save(filename)
-
+
if proc:
windowing.SetForegroundWindow(wnd)
-
+
# Send Alt-F4, then wait for process to end
keyboard.TypeString(r"{\4}", use_modifiers=True)
if not windowing.WaitForProcessExit(proc, timeout):
windowing.EndProcess(proc)
return "crashed"
-
+
if timedout:
return "timeout"
-
+
return None
def Time(urls, size, timeout, kwargs):
"""Measure how long it takes to load each of a series of URLs
-
+
Args:
urls: list of URLs to time
size: size of browser window to use
timeout: amount of time to wait for page to load
kwargs: miscellaneous keyword args
-
+
Returns:
A list of tuples (url, time). "time" can be "crashed" or "timeout"
"""
if "path" in kwargs and kwargs["path"]: path = kwargs["path"]
else: path = DEFAULT_PATH
proc = None
-
+
# Visit each URL we're given
if type(urls) in types.StringTypes: urls = [urls]
-
+
ret = []
for url in urls:
try:
# Invoke the browser if necessary
if not proc:
(wnd, proc, address_bar, render_pane) = InvokeBrowser(path)
-
+
# Resize and reposition the frame
windowing.MoveAndSizeWindow(wnd, (0,0), size, render_pane)
-
+
# Double-click in the address bar, type the name, and press Enter
mouse.ClickInWindow(address_bar)
keyboard.TypeString(url, 0.1)
keyboard.TypeString("\n")
-
+
# Wait for the page to finish loading
load_time = windowing.WaitForThrobber(wnd, (20, 16, 36, 32), timeout)
-
+
timedout = load_time < 0
-
+
if timedout:
load_time = "timeout"
-
+
# Send an alt-F4 to make the browser close; if this times out,
# we've probably got a crash
windowing.SetForegroundWindow(wnd)
-
+
keyboard.TypeString(r"{\4}", use_modifiers=True)
if not windowing.WaitForProcessExit(proc, timeout):
windowing.EndProcess(proc)
@@ -167,10 +167,10 @@ def Time(urls, size, timeout, kwargs):
except pywintypes.error:
proc = None
load_time = "crashed"
-
+
ret.append( (url, load_time) )
- if proc:
+ if proc:
windowing.SetForegroundWindow(wnd)
keyboard.TypeString(r"{\4}", use_modifiers=True)
if not windowing.WaitForProcessExit(proc, timeout):
@@ -183,7 +183,7 @@ if __name__ == "__main__":
# We're being invoked rather than imported, so run some tests
path = r"c:\sitecompare\scrapes\chrome\0.1.97.0"
windowing.PreparePath(path)
-
+
# Scrape three sites and save the results
Scrape([
"http://www.microsoft.com",
diff --git a/tools/site_compare/scrapers/firefox/__init__.py b/tools/site_compare/scrapers/firefox/__init__.py
index 255dc4b..7eb9291 100644
--- a/tools/site_compare/scrapers/firefox/__init__.py
+++ b/tools/site_compare/scrapers/firefox/__init__.py
@@ -3,29 +3,29 @@
# Copyright 2007 Google Inc. All Rights Reserved.
"""Selects the appropriate scraper for Firefox."""
-
+
__author__ = 'jhaas@google.com (Jonathan Haas)'
def GetScraper(version):
"""Returns the scraper module for the given version.
-
+
Args:
version: version string of IE, or None for most recent
-
+
Returns:
scrape module for given version
"""
-
+
# Pychecker will warn that the parameter is unused; we only
# support one version of Firefox at this time
-
+
# We only have one version of the Firefox scraper for now
return __import__("firefox2", globals(), locals(), [''])
-
+
# if invoked rather than imported, test
if __name__ == "__main__":
version = "2.0.0.6"
-
+
print GetScraper("2.0.0.6").version
- \ No newline at end of file
+
diff --git a/tools/site_compare/scrapers/firefox/firefox2.py b/tools/site_compare/scrapers/firefox/firefox2.py
index 0fdec98..fa0d620 100644
--- a/tools/site_compare/scrapers/firefox/firefox2.py
+++ b/tools/site_compare/scrapers/firefox/firefox2.py
@@ -29,29 +29,29 @@ def GetBrowser(path):
Args:
path: full path to browser
-
+
Returns:
A tuple of (process handle, render pane)
"""
if not path: path = DEFAULT_PATH
-
+
# Invoke Firefox
(proc, wnd) = windowing.InvokeAndWait(path)
-
+
# Get the content pane
render_pane = windowing.FindChildWindow(
wnd,
"MozillaWindowClass/MozillaWindowClass/MozillaWindowClass")
-
+
return (proc, wnd, render_pane)
def InvokeBrowser(path):
"""Invoke the Firefox browser.
-
+
Args:
path: full path to browser
-
+
Returns:
A tuple of (main window, process handle, render pane)
"""
@@ -64,18 +64,18 @@ def InvokeBrowser(path):
else:
# Invoke Firefox
(proc, wnd) = windowing.InvokeAndWait(path)
-
+
# Get the content pane
render_pane = windowing.FindChildWindow(
wnd,
"MozillaWindowClass/MozillaWindowClass/MozillaWindowClass")
-
+
return (wnd, proc, render_pane)
-
+
def Scrape(urls, outdir, size, pos, timeout=20, **kwargs):
"""Invoke a browser, send it to a series of URLs, and save its output.
-
+
Args:
urls: list of URLs to scrape
outdir: directory to place output
@@ -83,7 +83,7 @@ def Scrape(urls, outdir, size, pos, timeout=20, **kwargs):
pos: position of browser window
timeout: amount of time to wait for page to load
kwargs: miscellaneous keyword args
-
+
Returns:
None if success, else an error string
"""
@@ -91,28 +91,28 @@ def Scrape(urls, outdir, size, pos, timeout=20, **kwargs):
else: path = DEFAULT_PATH
(wnd, proc, render_pane) = InvokeBrowser(path)
-
+
# Resize and reposition the frame
windowing.MoveAndSizeWindow(wnd, pos, size, render_pane)
-
+
time.sleep(3)
-
+
# Firefox is a bit of a pain: it doesn't use standard edit controls,
- # and it doesn't display a throbber when there's no tab. Let's make
+ # and it doesn't display a throbber when there's no tab. Let's make
# sure there's at least one tab, then select the first one
-
+
mouse.ClickInWindow(wnd)
keyboard.TypeString("[t]", True)
mouse.ClickInWindow(wnd, (30, 115))
time.sleep(2)
timedout = False
-
+
# Visit each URL we're given
if type(urls) in types.StringTypes: urls = [urls]
-
+
for url in urls:
-
+
# Use keyboard shortcuts
keyboard.TypeString("{d}", True)
keyboard.TypeString(url)
@@ -124,10 +124,10 @@ def Scrape(urls, outdir, size, pos, timeout=20, **kwargs):
if timedout:
break
-
+
# Scrape the page
image = windowing.ScrapeWindow(render_pane)
-
+
# Save to disk
if "filename" in kwargs:
if callable(kwargs["filename"]):
@@ -137,58 +137,58 @@ def Scrape(urls, outdir, size, pos, timeout=20, **kwargs):
else:
filename = windowing.URLtoFilename(url, outdir, ".bmp")
image.save(filename)
-
+
# Close all the tabs, cheesily
mouse.ClickInWindow(wnd)
-
+
while len(windowing.FindChildWindows(0, "MozillaUIWindowClass")):
keyboard.TypeString("[w]", True)
time.sleep(1)
-
+
if timedout:
return "timeout"
def Time(urls, size, timeout, **kwargs):
"""Measure how long it takes to load each of a series of URLs
-
+
Args:
urls: list of URLs to time
size: size of browser window to use
timeout: amount of time to wait for page to load
kwargs: miscellaneous keyword args
-
+
Returns:
A list of tuples (url, time). "time" can be "crashed" or "timeout"
"""
if "path" in kwargs and kwargs["path"]: path = kwargs["path"]
else: path = DEFAULT_PATH
proc = None
-
+
# Visit each URL we're given
if type(urls) in types.StringTypes: urls = [urls]
-
+
ret = []
for url in urls:
try:
# Invoke the browser if necessary
if not proc:
(wnd, proc, render_pane) = InvokeBrowser(path)
-
+
# Resize and reposition the frame
windowing.MoveAndSizeWindow(wnd, (0,0), size, render_pane)
time.sleep(3)
-
+
# Firefox is a bit of a pain: it doesn't use standard edit controls,
- # and it doesn't display a throbber when there's no tab. Let's make
+ # and it doesn't display a throbber when there's no tab. Let's make
# sure there's at least one tab, then select the first one
-
+
mouse.ClickInWindow(wnd)
keyboard.TypeString("[t]", True)
mouse.ClickInWindow(wnd, (30, 115))
time.sleep(2)
-
+
# Use keyboard shortcuts
keyboard.TypeString("{d}", True)
keyboard.TypeString(url)
@@ -197,34 +197,34 @@ def Time(urls, size, timeout, **kwargs):
# Wait for the page to finish loading
load_time = windowing.WaitForThrobber(wnd, (10, 96, 26, 112), timeout)
timedout = load_time < 0
-
+
if timedout:
load_time = "timeout"
-
+
# Try to close the browser; if this fails it's probably a crash
mouse.ClickInWindow(wnd)
-
+
count = 0
- while (len(windowing.FindChildWindows(0, "MozillaUIWindowClass"))
+ while (len(windowing.FindChildWindows(0, "MozillaUIWindowClass"))
and count < 5):
keyboard.TypeString("[w]", True)
time.sleep(1)
count = count + 1
-
+
if len(windowing.FindChildWindows(0, "MozillaUIWindowClass")):
windowing.EndProcess(proc)
load_time = "crashed"
-
+
proc = None
except pywintypes.error:
proc = None
load_time = "crashed"
-
+
ret.append( (url, load_time) )
-
+
if proc:
count = 0
- while (len(windowing.FindChildWindows(0, "MozillaUIWindowClass"))
+ while (len(windowing.FindChildWindows(0, "MozillaUIWindowClass"))
and count < 5):
keyboard.TypeString("[w]", True)
time.sleep(1)
@@ -236,7 +236,7 @@ if __name__ == "__main__":
# We're being invoked rather than imported, so run some tests
path = r"c:\sitecompare\scrapes\Firefox\2.0.0.6"
windowing.PreparePath(path)
-
+
# Scrape three sites and save the results
Scrape(
["http://www.microsoft.com", "http://www.google.com",
diff --git a/tools/site_compare/scrapers/ie/__init__.py b/tools/site_compare/scrapers/ie/__init__.py
index 4b8949b..8fb95db 100644
--- a/tools/site_compare/scrapers/ie/__init__.py
+++ b/tools/site_compare/scrapers/ie/__init__.py
@@ -9,23 +9,23 @@ __author__ = 'jhaas@google.com (Jonathan Haas)'
def GetScraper(version):
"""Returns the scraper module for the given version.
-
+
Args:
version: version string of IE, or None for most recent
-
+
Returns:
scrape module for given version
"""
# Pychecker will warn that the parameter is unused; we only
# support one version of IE at this time
-
+
# We only have one version of the IE scraper for now
return __import__("ie7", globals(), locals(), [''])
-
+
# if invoked rather than imported, test
if __name__ == "__main__":
version = "7.0.5370.1"
-
+
print GetScraper(version).version
- \ No newline at end of file
+
diff --git a/tools/site_compare/scrapers/ie/ie7.py b/tools/site_compare/scrapers/ie/ie7.py
index f5d7583..da26d9b 100644
--- a/tools/site_compare/scrapers/ie/ie7.py
+++ b/tools/site_compare/scrapers/ie/ie7.py
@@ -23,29 +23,29 @@ def GetBrowser(path):
Args:
path: full path to browser
-
+
Returns:
A tuple of (process handle, render pane)
"""
if not path: path = DEFAULT_PATH
-
+
(iewnd, ieproc, address_bar, render_pane, tab_window) = InvokeBrowser(path)
return (ieproc, iewnd, render_pane)
def InvokeBrowser(path):
"""Invoke the IE browser.
-
+
Args:
path: full path to browser
-
+
Returns:
A tuple of (main window, process handle, address bar,
render_pane, tab_window)
"""
# Invoke IE
(ieproc, iewnd) = windowing.InvokeAndWait(path)
-
+
# Get windows we'll need
for tries in xrange(10):
try:
@@ -60,13 +60,13 @@ def InvokeBrowser(path):
time.sleep(1)
continue
break
-
+
return (iewnd, ieproc, address_bar, render_pane, tab_window)
def Scrape(urls, outdir, size, pos, timeout=20, **kwargs):
"""Invoke a browser, send it to a series of URLs, and save its output.
-
+
Args:
urls: list of URLs to scrape
outdir: directory to place output
@@ -74,32 +74,32 @@ def Scrape(urls, outdir, size, pos, timeout=20, **kwargs):
pos: position of browser window
timeout: amount of time to wait for page to load
kwargs: miscellaneous keyword args
-
+
Returns:
None if success, else an error string
"""
path = r"c:\program files\internet explorer\iexplore.exe"
-
+
if "path" in kwargs and kwargs["path"]: path = kwargs["path"]
(iewnd, ieproc, address_bar, render_pane, tab_window) = (
InvokeBrowser(path) )
-
+
# Resize and reposition the frame
windowing.MoveAndSizeWindow(iewnd, pos, size, render_pane)
-
+
# Visit each URL we're given
if type(urls) in types.StringTypes: urls = [urls]
-
+
timedout = False
-
+
for url in urls:
-
+
# Double-click in the address bar, type the name, and press Enter
mouse.DoubleClickInWindow(address_bar)
keyboard.TypeString(url)
keyboard.TypeString("\n")
-
+
# Wait for the page to finish loading
load_time = windowing.WaitForThrobber(
tab_window, (6, 8, 22, 24), timeout)
@@ -107,10 +107,10 @@ def Scrape(urls, outdir, size, pos, timeout=20, **kwargs):
if timedout:
break
-
+
# Scrape the page
image = windowing.ScrapeWindow(render_pane)
-
+
# Save to disk
if "filename" in kwargs:
if callable(kwargs["filename"]):
@@ -120,55 +120,55 @@ def Scrape(urls, outdir, size, pos, timeout=20, **kwargs):
else:
filename = windowing.URLtoFilename(url, outdir, ".bmp")
image.save(filename)
-
+
windowing.EndProcess(ieproc)
-
+
if timedout:
return "timeout"
-
-
+
+
def Time(urls, size, timeout, **kwargs):
"""Measure how long it takes to load each of a series of URLs
-
+
Args:
urls: list of URLs to time
size: size of browser window to use
timeout: amount of time to wait for page to load
kwargs: miscellaneous keyword args
-
+
Returns:
A list of tuples (url, time). "time" can be "crashed" or "timeout"
"""
if "path" in kwargs and kwargs["path"]: path = kwargs["path"]
else: path = DEFAULT_PATH
proc = None
-
+
# Visit each URL we're given
if type(urls) in types.StringTypes: urls = [urls]
-
+
ret = []
for url in urls:
try:
# Invoke the browser if necessary
if not proc:
(wnd, proc, address_bar, render_pane, tab_window) = InvokeBrowser(path)
-
+
# Resize and reposition the frame
windowing.MoveAndSizeWindow(wnd, (0,0), size, render_pane)
-
+
# Double-click in the address bar, type the name, and press Enter
mouse.DoubleClickInWindow(address_bar)
keyboard.TypeString(url)
keyboard.TypeString("\n")
-
+
# Wait for the page to finish loading
load_time = windowing.WaitForThrobber(
tab_window, (6, 8, 22, 24), timeout)
timedout = load_time < 0
-
+
if timedout:
load_time = "timeout"
-
+
# Send an alt-F4 to make the browser close; if this times out,
# we've probably got a crash
keyboard.TypeString(r"{\4}", use_modifiers=True)
@@ -179,9 +179,9 @@ def Time(urls, size, timeout, **kwargs):
except pywintypes.error:
load_time = "crashed"
proc = None
-
+
ret.append( (url, load_time) )
-
+
# Send an alt-F4 to make the browser close; if this times out,
# we've probably got a crash
if proc:
@@ -191,7 +191,7 @@ def Time(urls, size, timeout, **kwargs):
return ret
-
+
if __name__ == "__main__":
# We're being invoked rather than imported, so run some tests
path = r"c:\sitecompare\scrapes\ie7\7.0.5380.11"
diff --git a/tools/site_compare/site_compare.py b/tools/site_compare/site_compare.py
index 976f0ef..15359fa 100644
--- a/tools/site_compare/site_compare.py
+++ b/tools/site_compare/site_compare.py
@@ -37,7 +37,7 @@ import commands.scrape # scrape a URL or series of URLs to a bitmap
def Scrape(browsers, urls, window_size=(1024, 768),
window_pos=(0, 0), timeout=20, save_path=None, **kwargs):
"""Invoke one or more browsers over one or more URLs, scraping renders.
-
+
Args:
browsers: browsers to invoke with optional version strings
urls: URLs to visit
@@ -49,43 +49,43 @@ def Scrape(browsers, urls, window_size=(1024, 768),
kwargs: miscellaneous keyword args, passed to scraper
Returns:
None
-
+
@TODO(jhaas): more parameters, or perhaps an indefinite dictionary
parameter, for things like length of time to wait for timeout, speed
of mouse clicks, etc. Possibly on a per-browser, per-URL, or
per-browser-per-URL basis
"""
-
+
if type(browsers) in types.StringTypes: browsers = [browsers]
-
+
if save_path is None:
# default save path is "scrapes" off the current root
save_path = os.path.join(os.path.split(__file__)[0], "Scrapes")
-
+
for browser in browsers:
# Browsers should be tuples of (browser, version)
if type(browser) in types.StringTypes: browser = (browser, None)
scraper = scrapers.GetScraper(browser)
-
+
full_path = os.path.join(save_path, browser[0], scraper.version)
drivers.windowing.PreparePath(full_path)
-
+
scraper.Scrape(urls, full_path, window_size, window_pos, timeout, kwargs)
-
-
+
+
def Compare(base, compare, ops, root_path=None, out_path=None):
"""Compares a series of scrapes using a series of operators.
-
+
Args:
base: (browser, version) tuple of version to consider the baseline
compare: (browser, version) tuple of version to compare to
ops: list of operators plus operator arguments
root_path: root of the scrapes
out_path: place to put any output from the operators
-
+
Returns:
None
-
+
@TODO(jhaas): this method will likely change, to provide a robust and
well-defined way of chaining operators, applying operators conditionally,
and full-featured scripting of the operator chain. There also needs
@@ -95,28 +95,28 @@ def Compare(base, compare, ops, root_path=None, out_path=None):
if root_path is None:
# default save path is "scrapes" off the current root
root_path = os.path.join(os.path.split(__file__)[0], "Scrapes")
-
+
if out_path is None:
out_path = os.path.join(os.path.split(__file__)[0], "Compares")
-
+
if type(base) in types.StringTypes: base = (base, None)
if type(compare) in types.StringTypes: compare = (compare, None)
if type(ops) in types.StringTypes: ops = [ops]
-
+
base_dir = os.path.join(root_path, base[0])
compare_dir = os.path.join(root_path, compare[0])
-
+
if base[1] is None:
# base defaults to earliest capture
base = (base[0], max(os.listdir(base_dir)))
-
+
if compare[1] is None:
# compare defaults to latest capture
compare = (compare[0], min(os.listdir(compare_dir)))
-
+
out_path = os.path.join(out_path, base[0], base[1], compare[0], compare[1])
drivers.windowing.PreparePath(out_path)
-
+
# TODO(jhaas): right now we're just dumping output to a log file
# (and the console), which works as far as it goes but isn't nearly
# robust enough. Change this after deciding exactly what we want to
@@ -126,10 +126,10 @@ def Compare(base, compare, ops, root_path=None, out_path=None):
(base[0], base[1], compare[0], compare[1]))
out_file.write(description_string)
print description_string
-
+
base_dir = os.path.join(base_dir, base[1])
compare_dir = os.path.join(compare_dir, compare[1])
-
+
for filename in os.listdir(base_dir):
out_file.write("%s: " % filename)
@@ -137,15 +137,15 @@ def Compare(base, compare, ops, root_path=None, out_path=None):
out_file.write("Does not exist in target directory\n")
print "File %s does not exist in target directory" % filename
continue
-
+
base_filename = os.path.join(base_dir, filename)
compare_filename = os.path.join(compare_dir, filename)
-
+
for op in ops:
if type(op) in types.StringTypes: op = (op, None)
-
+
module = operators.GetOperator(op[0])
-
+
ret = module.Compare(base_filename, compare_filename)
if ret is None:
print "%s: OK" % (filename,)
@@ -154,24 +154,24 @@ def Compare(base, compare, ops, root_path=None, out_path=None):
print "%s: %s" % (filename, ret[0])
out_file.write("%s\n" % (ret[0]))
ret[1].save(os.path.join(out_path, filename))
-
+
out_file.close()
def main():
"""Main executable. Parse the command line and invoke the command."""
cmdline = command_line.CommandLine()
-
+
# The below two commands are currently unstable so have been disabled
# commands.compare2.CreateCommand(cmdline)
# commands.maskmaker.CreateCommand(cmdline)
commands.measure.CreateCommand(cmdline)
commands.scrape.CreateCommand(cmdline)
-
+
cmdline.ParseCommandLine()
if __name__ == "__main__":
main()
-
+
diff --git a/tools/site_compare/utils/browser_iterate.py b/tools/site_compare/utils/browser_iterate.py
index 50ed411..6ea4e5f 100644
--- a/tools/site_compare/utils/browser_iterate.py
+++ b/tools/site_compare/utils/browser_iterate.py
@@ -32,7 +32,7 @@ PORT = 42492
def SetupIterationCommandLine(cmd):
"""Adds the necessary flags for iteration to a command.
-
+
Args:
cmd: an object created by cmdline.AddCommand
"""
@@ -71,15 +71,15 @@ def SetupIterationCommandLine(cmd):
def Iterate(command, iteration_func):
"""Iterates over a list of URLs, calling a function on each.
-
+
Args:
command: the command line containing the iteration flags
iteration_func: called for each URL with (proc, wnd, url, result)
"""
-
+
# Retrieve the browser scraper to use to invoke the browser
scraper = scrapers.GetScraper((command["--browser"], command["--browserver"]))
-
+
def AttachToBrowser(path, timeout):
"""Invoke the browser process and connect to the socket."""
(proc, frame, wnd) = scraper.GetBrowser(path)
@@ -106,7 +106,7 @@ def Iterate(command, iteration_func):
if command["--size"]:
# Resize and reposition the frame
windowing.MoveAndSizeWindow(frame, (0, 0), command["--size"], wnd)
-
+
s.settimeout(timeout)
Iterate.proc = proc
@@ -133,7 +133,7 @@ def Iterate(command, iteration_func):
browser = command["--browserpath"]
else:
browser = None
-
+
# Read the URLs from the file
if command["--url"]:
url_list = [command["--url"]]
@@ -174,13 +174,13 @@ def Iterate(command, iteration_func):
try:
recv = Iterate.s.recv(MAX_URL)
response = response + recv
-
+
# Workaround for an oddity: when Firefox closes
# gracefully, somehow Python doesn't detect it.
# (Telnet does)
- if not recv:
+ if not recv:
raise socket.error
-
+
except socket.timeout:
response = url + ",hang\n"
DetachFromBrowser()
@@ -192,10 +192,10 @@ def Iterate(command, iteration_func):
# If we received a timeout response, restart the browser
if response[-9:] == ",timeout\n":
DetachFromBrowser()
-
+
# Invoke the iteration function
iteration_func(url, Iterate.proc, Iterate.wnd, response)
- # We're done
+ # We're done
DetachFromBrowser()