1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
|
#!/usr/bin/python
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be found
# in the LICENSE file.
"""Extracts registration forms from the corresponding HTML files.
Used for extracting forms within HTML files. This script is used in
conjunction with the webforms_aggregator.py script, which aggregates web pages
with fillable forms (i.e registration forms).
The purpose of this script is to extract out JavaScript elements that may be
causing parsing errors and timeout issues when running browser_tests.
Used as a standalone script but assumes that it is run from the directory in
which it is checked into.
Usage: forms_extractor.py [options]
Options:
-l LOG_LEVEL, --log_level=LOG_LEVEL,
LOG_LEVEL: debug, info, warning or error [default: error]
-h, --help show this help message and exit
"""
import glob
import logging
from optparse import OptionParser
import os
import re
import sys
class FormsExtractor(object):
"""Extracts HTML files, leaving only registration forms from the HTML file."""
HTML_FILES_PATTERN = r'*.html'
HTML_FILE_PREFIX = r'grabber-'
FORM_FILE_PREFIX = r'top100_'
REGISTRATION_PAGES_DIR = os.path.join(os.pardir, 'test', 'data', 'autofill',
'heuristics', 'input')
EXTRACTED_FORMS_DIR = os.path.join(os.pardir, 'test', 'data', 'autofill',
'heuristics', 'input')
logger = logging.getLogger(__name__)
log_handlers = {'StreamHandler': None}
# This pattern is used for removing all |<script>| code.
re_script_pattern = re.compile(
ur"""
<script # A new opening 'script' tag.
\b # The end of the word 'script'.
.*? # Any characters (non-greedy).
> # Ending of the opening tag '>'.
.*? # Any characters (non-greedy) between the tags.
</script\s*> # The '<script>' closing tag.
""", re.U | re.S | re.I | re.X)
# This pattern is used to remove all href js code.
re_href_js_pattern = re.compile(
ur"""
\bhref # The beginning of a new word.
\s*=\s* # Matches any spaces before and after the '=' symbol.
(?P<quote>[\'\"]) # Captures the single or double quote in the
# matching pattern.
\s*javascript\s* # Matches anything before and after 'javascript'.
.*? # Matches any characters (non-greedy) between the
# quotes.
\1 # Matches the previously captured single or double
# quotes.
""", re.U | re.S | re.I | re.X)
# This pattern is used for capturing js event |'onmouseover="..."'|
re_event = (
ur"""
\b # The beginning of a new word.
on\w+? # Matches all words starting with 'on' (non-greedy)
# |onmouseover|.
\s*=\s* # Matches any spaces before and after the '=' symbol.
(?P<quote>[\'\"]) # Captures the single or double quote in the matching
# pattern.
.*? # Matches any characters (non-greedy) between the
# quotes.
\1 # Matches the previously captured single or double
# quotes.
""")
# This pattern is used for removing code with js events, such as |onload|.
# By adding the leading |ur'<[^<>]*?'| and the trailing |'ur'[^<>]*?>'| the
# pattern matches to strings such as '<tr class="nav"
# onmouseover="mOvr1(this);" onmouseout="mOut1(this);">'
re_tag_with_events_pattern = re.compile(
ur"""
< # Matches character '<'.
[^<>]*? # Matches any characters except '<' and '>' (non-greedy).""" +
re_event +
ur"""
[^<>]*? # Matches any characters except '<' and '>' (non-greedy)
> # Matches character '>'.
""", re.U | re.S | re.I | re.X)
# Add white space characters at the end of the matched event. Leaves only the
# leading white space when substituted with empty string.
re_event_pattern = re.compile(re_event + ur'\s*', re.U | re.S | re.I | re.X)
def __init__(self, input_dir=REGISTRATION_PAGES_DIR,
output_dir=EXTRACTED_FORMS_DIR, logging_level=None):
"""Creates a FormsExtractor object.
Args:
input_dir: the directory of HTML files.
output_dir: the directory where the registration form files will be
saved.
logging_level: verbosity level, default is None.
Raises:
IOError exception if input directory doesn't exist.
"""
if logging_level:
if not self.log_handlers['StreamHandler']:
console = logging.StreamHandler()
console.setLevel(logging.DEBUG)
self.log_handlers['StreamHandler'] = console
self.logger.addHandler(console)
self.logger.setLevel(logging_level)
else:
if self.log_handlers['StreamHandler']:
self.logger.removeHandler(self.log_handlers['StreamHandler'])
self.log_handlers['StreamHandler'] = None
self._input_dir = input_dir
self._output_dir = output_dir
if not os.path.isdir(self._input_dir):
error_msg = 'Directory "%s" doesn\'t exist.' % self._input_dir
self.logger.error('Error: %s', error_msg)
raise IOError(error_msg)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
self._form_location_comment = ''
def _SubstituteAllEvents(self, matchobj):
"""Remove all js events that are present as attributes within a tag.
Args:
matchobj: A regexp |re.MatchObject| containing text that has at least one
event. Example: |<tr class="nav" onmouseover="mOvr1(this);"
onmouseout="mOut1(this);">|
Returns:
The text containing the tag with all the attributes except for the tags
with events. Example: |<tr class="nav">|
"""
tag_with_all_attrs = matchobj.group(0)
tag_with_all_attrs_except_events = self.re_event_pattern.sub(
'', tag_with_all_attrs)
return tag_with_all_attrs_except_events
def Extract(self):
"""Extracts and saves the extracted registration forms.
Iterates through all the HTML files.
"""
pathname_pattern = os.path.join(self._input_dir, self.HTML_FILES_PATTERN)
html_files = [f for f in glob.glob(pathname_pattern) if os.path.isfile(f)]
for filename in html_files:
self.logger.info('extracting file "%s" ...', filename)
with open(filename) as f:
html_content = self.re_tag_with_events_pattern.sub(
self._SubstituteAllEvents,
self.re_href_js_pattern.sub(
'', self.re_script_pattern.sub('', f.read())))
try:
with open(
os.path.join(self._output_dir, '%s%s' % (
self.FORM_FILE_PREFIX, os.path.split(filename)[1].replace(
self.HTML_FILE_PREFIX, '')) ), 'wb') as f:
f.write(html_content)
except IOError as e:
self.logger.error('Error: %s', e)
continue
self.logger.info('\tfile "%s" extracted SUCCESSFULLY!', filename)
def main():
# Command line options.
parser = OptionParser()
parser.add_option(
'-l', '--log_level', metavar='LOG_LEVEL', default='error',
help='LOG_LEVEL: debug, info, warning or error [default: %default]')
options, args = parser.parse_args()
options.log_level = options.log_level.upper()
if options.log_level not in ['DEBUG', 'INFO', 'WARNING', 'ERROR']:
print 'Wrong log_level argument.'
parser.print_help()
sys.exit(1)
# Convert the capitalized log level string into the actual argument used by
# the logging module.
options.log_level = getattr(logging, options.log_level)
extractor = FormsExtractor(logging_level=options.log_level)
extractor.Extract()
if __name__ == '__main__':
main()
|
