1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
|
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import logging
import time
import traceback
from app_yaml_helper import AppYamlHelper
from appengine_wrappers import (
GetAppVersion, DeadlineExceededError, IsDevServer, logservice)
from branch_utility import BranchUtility
from compiled_file_system import CompiledFileSystem
from empty_dir_file_system import EmptyDirFileSystem
from file_system_util import CreateURLsFromPaths
from github_file_system import GithubFileSystem
from host_file_system_creator import HostFileSystemCreator
from object_store_creator import ObjectStoreCreator
from render_servlet import RenderServlet
from server_instance import ServerInstance
from servlet import Servlet, Request, Response
import svn_constants
class _SingletonRenderServletDelegate(RenderServlet.Delegate):
def __init__(self, server_instance):
self._server_instance = server_instance
def CreateServerInstance(self):
return self._server_instance
class CronServlet(Servlet):
'''Servlet which runs a cron job.
'''
def __init__(self, request, delegate_for_test=None):
Servlet.__init__(self, request)
self._delegate = delegate_for_test or CronServlet.Delegate()
class Delegate(object):
'''CronServlet's runtime dependencies. Override for testing.
'''
def CreateBranchUtility(self, object_store_creator):
return BranchUtility.Create(object_store_creator)
def CreateHostFileSystemCreator(self, object_store_creator):
return HostFileSystemCreator(object_store_creator)
def CreateAppSamplesFileSystem(self, object_store_creator):
# TODO(kalman): CachingFileSystem wrapper for GithubFileSystem, but it's
# not supported yet (see comment there).
return (EmptyDirFileSystem() if IsDevServer() else
GithubFileSystem.Create(object_store_creator))
def GetAppVersion(self):
return GetAppVersion()
def Get(self):
# Crons often time out, and when they do *and* then eventually try to
# flush logs they die. Turn off autoflush and manually do so at the end.
logservice.AUTOFLUSH_ENABLED = False
try:
return self._GetImpl()
finally:
logservice.flush()
def _GetImpl(self):
# Cron strategy:
#
# Find all public template files and static files, and render them. Most of
# the time these won't have changed since the last cron run, so it's a
# little wasteful, but hopefully rendering is really fast (if it isn't we
# have a problem).
logging.info('cron: starting')
# This is returned every time RenderServlet wants to create a new
# ServerInstance.
server_instance = self._GetSafeServerInstance()
def get_via_render_servlet(path):
request = Request(path, self._request.host, self._request.headers)
delegate = _SingletonRenderServletDelegate(server_instance)
return RenderServlet(request, delegate).Get()
def run_cron_for_dir(d, path_prefix=''):
success = True
start_time = time.time()
files = dict(
CreateURLsFromPaths(server_instance.host_file_system, d, path_prefix))
logging.info('cron: rendering %s files from %s...' % (len(files), d))
try:
for i, path in enumerate(files):
error = None
try:
response = get_via_render_servlet(path)
if response.status != 200:
error = 'Got %s response' % response.status
except DeadlineExceededError:
logging.error(
'cron: deadline exceeded rendering %s (%s of %s): %s' % (
path, i + 1, len(files), traceback.format_exc()))
raise
except error:
pass
if error:
logging.error('cron: error rendering %s: %s' % (path, error))
success = False
finally:
logging.info('cron: rendering %s files from %s took %s seconds' % (
len(files), d, time.time() - start_time))
return success
success = True
try:
# Render all of the publicly accessible files.
cron_runs = [
# Note: rendering the public templates will pull in all of the private
# templates.
(svn_constants.PUBLIC_TEMPLATE_PATH, ''),
# Note: rendering the public templates will have pulled in the .js
# and manifest.json files (for listing examples on the API reference
# pages), but there are still images, CSS, etc.
(svn_constants.STATIC_PATH, 'static/'),
]
if not IsDevServer():
cron_runs.append(
(svn_constants.EXAMPLES_PATH, 'extensions/examples/'))
# Note: don't try to short circuit any of this stuff. We want to run
# the cron for all the directories regardless of intermediate
# failures.
for path, path_prefix in cron_runs:
success = run_cron_for_dir(path, path_prefix=path_prefix) and success
# TODO(kalman): Generic way for classes to request cron access. The next
# two special cases are ugly. It would potentially greatly speed up cron
# runs, too.
# Extension examples have zip files too. Well, so do apps, but the app
# file system doesn't get the Offline treatment so they don't need cron.
if not IsDevServer():
manifest_json = '/manifest.json'
example_zips = [
'%s.zip' % filename[:-len(manifest_json)]
for filename in server_instance.content_cache.GetFromFileListing(
svn_constants.EXAMPLES_PATH)
if filename.endswith(manifest_json)]
logging.info('cron: rendering %s example zips...' % len(example_zips))
start_time = time.time()
try:
success = success and all(
get_via_render_servlet('extensions/examples/%s' % z).status == 200
for z in example_zips)
finally:
logging.info('cron: rendering %s example zips took %s seconds' % (
len(example_zips), time.time() - start_time))
except DeadlineExceededError:
success = False
logging.info('cron: running Redirector cron...')
server_instance.redirector.Cron()
logging.info('cron: finished (%s)' % ('success' if success else 'failure',))
return (Response.Ok('Success') if success else
Response.InternalError('Failure'))
def _GetSafeServerInstance(self):
'''Returns a ServerInstance with a host file system at a safe revision,
meaning the last revision that the current running version of the server
existed.
'''
delegate = self._delegate
server_instance_at_head = self._CreateServerInstance(None)
app_yaml_handler = AppYamlHelper(
svn_constants.APP_YAML_PATH,
server_instance_at_head.host_file_system,
server_instance_at_head.object_store_creator,
server_instance_at_head.host_file_system_creator)
if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()):
# TODO(kalman): return a new ServerInstance at an explicit revision in
# case the HEAD version changes underneath us.
return server_instance_at_head
# The version in app.yaml is greater than the currently running app's.
# The safe version is the one before it changed.
safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan(
delegate.GetAppVersion()) - 1
logging.info('cron: app version %s is out of date, safe is %s' % (
delegate.GetAppVersion(), safe_revision))
return self._CreateServerInstance(safe_revision)
def _CreateServerInstance(self, revision):
object_store_creator = ObjectStoreCreator(start_empty=True)
branch_utility = self._delegate.CreateBranchUtility(object_store_creator)
host_file_system_creator = self._delegate.CreateHostFileSystemCreator(
object_store_creator)
host_file_system = host_file_system_creator.Create(revision=revision)
app_samples_file_system = self._delegate.CreateAppSamplesFileSystem(
object_store_creator)
compiled_host_fs_factory = CompiledFileSystem.Factory(
host_file_system,
object_store_creator)
return ServerInstance(object_store_creator,
host_file_system,
app_samples_file_system,
'',
compiled_host_fs_factory,
branch_utility,
host_file_system_creator)
|
