1 | # -*- coding: utf-8 -*- |
---|
2 | # |
---|
3 | # Copyright (C) 2007-2010 Edgewall Software |
---|
4 | # Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de> |
---|
5 | # All rights reserved. |
---|
6 | # |
---|
7 | # This software is licensed as described in the file COPYING, which |
---|
8 | # you should have received as part of this distribution. The terms |
---|
9 | # are also available at http://bitten.edgewall.org/wiki/License. |
---|
10 | |
---|
11 | """Build master implementation.""" |
---|
12 | |
---|
13 | import calendar |
---|
14 | import re |
---|
15 | import time |
---|
16 | from StringIO import StringIO |
---|
17 | |
---|
18 | from trac.attachment import Attachment |
---|
19 | from trac.config import BoolOption, IntOption, Option |
---|
20 | from trac.core import * |
---|
21 | from trac.resource import ResourceNotFound |
---|
22 | from trac.web import IRequestHandler, RequestDone |
---|
23 | |
---|
24 | from bitten import PROTOCOL_VERSION |
---|
25 | from bitten.model import BuildConfig, Build, BuildStep, BuildLog, Report, \ |
---|
26 | TargetPlatform |
---|
27 | |
---|
28 | from bitten.main import BuildSystem |
---|
29 | from bitten.queue import BuildQueue |
---|
30 | from bitten.recipe import Recipe |
---|
31 | from bitten.util import xmlio |
---|
32 | from bitten.util.repository import get_repos |
---|
33 | |
---|
34 | |
---|
35 | __all__ = ['BuildMaster'] |
---|
36 | __docformat__ = 'restructuredtext en' |
---|
37 | |
---|
38 | |
---|
39 | HTTP_BAD_REQUEST = 400 |
---|
40 | HTTP_FORBIDDEN = 403 |
---|
41 | HTTP_NOT_FOUND = 404 |
---|
42 | HTTP_METHOD_NOT_ALLOWED = 405 |
---|
43 | HTTP_CONFLICT = 409 |
---|
44 | |
---|
45 | |
---|
46 | class BuildMaster(Component): |
---|
47 | """Trac request handler implementation for the build master.""" |
---|
48 | |
---|
49 | implements(IRequestHandler) |
---|
50 | |
---|
51 | # Configuration options |
---|
52 | |
---|
53 | adjust_timestamps = BoolOption('bitten', 'adjust_timestamps', False, doc= |
---|
54 | """Whether the timestamps of builds should be adjusted to be close |
---|
55 | to the timestamps of the corresponding changesets.""") |
---|
56 | |
---|
57 | build_all = BoolOption('bitten', 'build_all', False, doc= |
---|
58 | """Whether to request builds of older revisions even if a younger |
---|
59 | revision has already been built.""") |
---|
60 | |
---|
61 | stabilize_wait = IntOption('bitten', 'stabilize_wait', 0, doc= |
---|
62 | """The time in seconds to wait for the repository to stabilize before |
---|
63 | queuing up a new build. This allows time for developers to check in |
---|
64 | a group of related changes back to back without spawning multiple |
---|
65 | builds.""") |
---|
66 | |
---|
67 | slave_timeout = IntOption('bitten', 'slave_timeout', 3600, doc= |
---|
68 | """The time in seconds after which a build is cancelled if the slave |
---|
69 | does not report progress.""") |
---|
70 | |
---|
71 | logs_dir = Option('bitten', 'logs_dir', "log/bitten", doc= |
---|
72 | """The directory on the server in which client log files will be stored.""") |
---|
73 | |
---|
74 | quick_status = BoolOption('bitten', 'quick_status', False, doc= |
---|
75 | """Whether to show the current build status within the Trac main |
---|
76 | navigation bar. '''Note:''' The feature requires expensive database and |
---|
77 | repository checks for every page request, and should not be enabled |
---|
78 | if the project has a large repository or uses a non-Subversion |
---|
79 | repository such as Mercurial or Git.""") |
---|
80 | |
---|
81 | def __init__(self): |
---|
82 | self.env.systeminfo.append(('Bitten', |
---|
83 | __import__('bitten', ['__version__']).__version__)) |
---|
84 | |
---|
85 | # IRequestHandler methods |
---|
86 | |
---|
87 | def match_request(self, req): |
---|
88 | match = re.match(r'/builds(?:/(\d+)(?:/(\w+)/([^/]+)?)?)?$', |
---|
89 | req.path_info) |
---|
90 | if match: |
---|
91 | if match.group(1): |
---|
92 | req.args['id'] = match.group(1) |
---|
93 | req.args['collection'] = match.group(2) |
---|
94 | req.args['member'] = match.group(3) |
---|
95 | return True |
---|
96 | |
---|
97 | def process_request(self, req): |
---|
98 | req.perm.assert_permission('BUILD_EXEC') |
---|
99 | |
---|
100 | if 'trac_auth' in req.incookie: |
---|
101 | slave_token = req.incookie['trac_auth'].value |
---|
102 | else: |
---|
103 | slave_token = req.session.sid |
---|
104 | |
---|
105 | if 'id' not in req.args: |
---|
106 | if req.method != 'POST': |
---|
107 | self._send_response(req, |
---|
108 | body='Only POST allowed for build creation.') |
---|
109 | return self._process_build_creation(req, slave_token) |
---|
110 | |
---|
111 | build = Build.fetch(self.env, req.args['id']) |
---|
112 | if not build: |
---|
113 | self._send_error(req, HTTP_NOT_FOUND, |
---|
114 | 'No such build (%s)' % req.args['id']) |
---|
115 | |
---|
116 | build_token = build.slave_info.get('token', '') |
---|
117 | if build_token != slave_token: |
---|
118 | self._send_error(req, HTTP_CONFLICT, |
---|
119 | 'Token mismatch (wrong slave): slave=%s, build=%s' \ |
---|
120 | % (slave_token, build_token)) |
---|
121 | |
---|
122 | config = BuildConfig.fetch(self.env, build.config) |
---|
123 | |
---|
124 | if not req.args['collection']: |
---|
125 | if req.method == 'DELETE': |
---|
126 | return self._process_build_cancellation(req, config, build) |
---|
127 | else: |
---|
128 | return self._process_build_initiation(req, config, build) |
---|
129 | |
---|
130 | if req.method != 'POST': |
---|
131 | self._send_error(req, HTTP_METHOD_NOT_ALLOWED, |
---|
132 | 'Method %s not allowed' % req.method) |
---|
133 | |
---|
134 | if req.args['collection'] == 'steps': |
---|
135 | return self._process_build_step(req, config, build) |
---|
136 | elif req.args['collection'] == 'attach': |
---|
137 | return self._process_attachment(req, config, build) |
---|
138 | elif req.args['collection'] == 'keepalive': |
---|
139 | return self._process_keepalive(req, config, build) |
---|
140 | else: |
---|
141 | self._send_error(req, HTTP_NOT_FOUND, |
---|
142 | "No such collection '%s'" % req.args['collection']) |
---|
143 | |
---|
144 | # Internal methods |
---|
145 | |
---|
146 | def _send_response(self, req, code=200, body='', headers=None): |
---|
147 | """ Formats and sends the response, raising ``RequestDone``. """ |
---|
148 | if isinstance(body, unicode): |
---|
149 | body = body.encode('utf-8') |
---|
150 | req.send_response(code) |
---|
151 | headers = headers or {} |
---|
152 | headers.setdefault('Content-Length', len(body)) |
---|
153 | for header in headers: |
---|
154 | req.send_header(header, headers[header]) |
---|
155 | req.write(body) |
---|
156 | raise RequestDone |
---|
157 | |
---|
158 | def _send_error(self, req, code=500, message=''): |
---|
159 | """ Formats and sends the error, raising ``RequestDone``. """ |
---|
160 | headers = {'Content-Type': 'text/plain', |
---|
161 | 'Content-Length': str(len(message))} |
---|
162 | self._send_response(req, code, body=message, headers=headers) |
---|
163 | |
---|
164 | def _process_build_creation(self, req, slave_token): |
---|
165 | queue = BuildQueue(self.env, build_all=self.build_all, |
---|
166 | stabilize_wait=self.stabilize_wait, |
---|
167 | timeout=self.slave_timeout) |
---|
168 | try: |
---|
169 | queue.populate() |
---|
170 | except AssertionError, e: |
---|
171 | self.log.error(e.message, exc_info=True) |
---|
172 | self._send_error(req, HTTP_BAD_REQUEST, e.message) |
---|
173 | |
---|
174 | try: |
---|
175 | elem = xmlio.parse(req.read()) |
---|
176 | except xmlio.ParseError, e: |
---|
177 | self.log.error('Error parsing build initialization request: %s', e, |
---|
178 | exc_info=True) |
---|
179 | self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error') |
---|
180 | |
---|
181 | slave_version = int(elem.attr.get('version', 1)) |
---|
182 | |
---|
183 | # FIXME: Remove version compatibility code. |
---|
184 | # The initial difference between protocol version 3 and 4 is that |
---|
185 | # the master allows keepalive requests-- the master must be |
---|
186 | # at least 4 before slaves supporting version 4 are allowed. When |
---|
187 | # the first force master/slave upgrade requirement comes in |
---|
188 | # (or we bump the) version number again, remove this code. |
---|
189 | if slave_version == 3 and PROTOCOL_VERSION == 4: |
---|
190 | self.log.info('Allowing slave version %d to process build for ' |
---|
191 | 'compatibility. Upgrade slave to support build ' |
---|
192 | 'keepalives.', slave_version) |
---|
193 | elif slave_version != PROTOCOL_VERSION: |
---|
194 | self._send_error(req, HTTP_BAD_REQUEST, |
---|
195 | "Master-Slave version mismatch: master=%d, slave=%d" % \ |
---|
196 | (PROTOCOL_VERSION, slave_version)) |
---|
197 | |
---|
198 | slavename = elem.attr['name'] |
---|
199 | properties = {'name': slavename, Build.IP_ADDRESS: req.remote_addr, |
---|
200 | Build.TOKEN: slave_token} |
---|
201 | self.log.info('Build slave %r connected from %s with token %s', |
---|
202 | slavename, req.remote_addr, slave_token) |
---|
203 | |
---|
204 | for child in elem.children(): |
---|
205 | if child.name == 'platform': |
---|
206 | properties[Build.MACHINE] = child.gettext() |
---|
207 | properties[Build.PROCESSOR] = child.attr.get('processor') |
---|
208 | elif child.name == 'os': |
---|
209 | properties[Build.OS_NAME] = child.gettext() |
---|
210 | properties[Build.OS_FAMILY] = child.attr.get('family') |
---|
211 | properties[Build.OS_VERSION] = child.attr.get('version') |
---|
212 | elif child.name == 'package': |
---|
213 | for name, value in child.attr.items(): |
---|
214 | if name == 'name': |
---|
215 | continue |
---|
216 | properties[child.attr['name'] + '.' + name] = value |
---|
217 | |
---|
218 | self.log.debug('Build slave configuration: %r', properties) |
---|
219 | |
---|
220 | build = queue.get_build_for_slave(slavename, properties) |
---|
221 | if not build: |
---|
222 | self._send_response(req, 204, '', {}) |
---|
223 | |
---|
224 | self._send_response(req, 201, 'Build pending', headers={ |
---|
225 | 'Content-Type': 'text/plain', |
---|
226 | 'Location': req.abs_href.builds(build.id)}) |
---|
227 | |
---|
228 | def _process_build_cancellation(self, req, config, build): |
---|
229 | self.log.info('Build slave %r cancelled build %d', build.slave, |
---|
230 | build.id) |
---|
231 | build.status = Build.PENDING |
---|
232 | build.slave = None |
---|
233 | build.slave_info = {} |
---|
234 | build.started = 0 |
---|
235 | db = self.env.get_db_cnx() |
---|
236 | for step in list(BuildStep.select(self.env, build=build.id, db=db)): |
---|
237 | step.delete(db=db) |
---|
238 | build.update(db=db) |
---|
239 | |
---|
240 | Attachment.delete_all(self.env, 'build', build.resource.id, db) |
---|
241 | |
---|
242 | db.commit() |
---|
243 | |
---|
244 | for listener in BuildSystem(self.env).listeners: |
---|
245 | listener.build_aborted(build) |
---|
246 | |
---|
247 | self._send_response(req, 204, '', {}) |
---|
248 | |
---|
249 | def _process_build_initiation(self, req, config, build): |
---|
250 | self.log.info('Build slave %r initiated build %d', build.slave, |
---|
251 | build.id) |
---|
252 | build.started = int(time.time()) |
---|
253 | build.last_activity = build.started |
---|
254 | build.update() |
---|
255 | |
---|
256 | for listener in BuildSystem(self.env).listeners: |
---|
257 | listener.build_started(build) |
---|
258 | |
---|
259 | repos_name, repos, repos_path = get_repos( |
---|
260 | self.env, config.path, req.authname) |
---|
261 | xml = xmlio.parse(config.recipe) |
---|
262 | xml.attr['path'] = config.path |
---|
263 | xml.attr['revision'] = build.rev |
---|
264 | xml.attr['config'] = config.name |
---|
265 | xml.attr['build'] = str(build.id) |
---|
266 | target_platform = TargetPlatform.fetch(self.env, build.platform) |
---|
267 | xml.attr['platform'] = target_platform.name |
---|
268 | xml.attr['name'] = build.slave |
---|
269 | xml.attr['form_token'] = req.form_token # For posting attachments |
---|
270 | xml.attr['reponame'] = repos_name != '(default)' and repos_name or '' |
---|
271 | xml.attr['repopath'] = repos_path.strip('/') |
---|
272 | body = str(xml) |
---|
273 | |
---|
274 | self.log.info('Build slave %r initiated build %d', build.slave, |
---|
275 | build.id) |
---|
276 | |
---|
277 | # create the first step, mark it as in-progress. |
---|
278 | |
---|
279 | recipe = Recipe(xmlio.parse(config.recipe)) |
---|
280 | stepname = recipe.__iter__().next().id |
---|
281 | |
---|
282 | step = self._start_new_step(build, stepname) |
---|
283 | step.insert() |
---|
284 | |
---|
285 | self._send_response(req, 200, body, headers={ |
---|
286 | 'Content-Type': 'application/x-bitten+xml', |
---|
287 | 'Content-Length': str(len(body)), |
---|
288 | 'Content-Disposition': |
---|
289 | 'attachment; filename=recipe_%s_r%s.xml' % |
---|
290 | (config.name, build.rev)}) |
---|
291 | |
---|
292 | def _process_build_step(self, req, config, build): |
---|
293 | try: |
---|
294 | elem = xmlio.parse(req.read()) |
---|
295 | except xmlio.ParseError, e: |
---|
296 | self.log.error('Error parsing build step result: %s', e, |
---|
297 | exc_info=True) |
---|
298 | self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error') |
---|
299 | stepname = elem.attr['step'] |
---|
300 | |
---|
301 | # we should have created this step previously; if it hasn't, |
---|
302 | # the master and slave are processing steps out of order. |
---|
303 | step = BuildStep.fetch(self.env, build=build.id, name=stepname) |
---|
304 | if not step: |
---|
305 | self._send_error(req, HTTP_CONFLICT, 'Build step has not been created.') |
---|
306 | |
---|
307 | recipe = Recipe(xmlio.parse(config.recipe)) |
---|
308 | index = None |
---|
309 | current_step = None |
---|
310 | for num, recipe_step in enumerate(recipe): |
---|
311 | if recipe_step.id == stepname: |
---|
312 | index = num |
---|
313 | current_step = recipe_step |
---|
314 | if index is None: |
---|
315 | self._send_error(req, HTTP_FORBIDDEN, |
---|
316 | 'No such build step' % stepname) |
---|
317 | last_step = index == num |
---|
318 | |
---|
319 | self.log.debug('Slave %s (build %d) completed step %d (%s) with ' |
---|
320 | 'status %s', build.slave, build.id, index, stepname, |
---|
321 | elem.attr['status']) |
---|
322 | |
---|
323 | db = self.env.get_db_cnx() |
---|
324 | |
---|
325 | step.stopped = int(time.time()) |
---|
326 | |
---|
327 | if elem.attr['status'] == 'failure': |
---|
328 | self.log.warning('Build %s step %s failed', build.id, stepname) |
---|
329 | step.status = BuildStep.FAILURE |
---|
330 | if current_step.onerror == 'fail': |
---|
331 | last_step = True |
---|
332 | else: |
---|
333 | step.status = BuildStep.SUCCESS |
---|
334 | step.errors += [error.gettext() for error in elem.children('error')] |
---|
335 | |
---|
336 | # TODO: step.update(db=db) |
---|
337 | step.delete(db=db) |
---|
338 | step.insert(db=db) |
---|
339 | |
---|
340 | # Collect log messages from the request body |
---|
341 | for idx, log_elem in enumerate(elem.children('log')): |
---|
342 | build_log = BuildLog(self.env, build=build.id, step=stepname, |
---|
343 | generator=log_elem.attr.get('generator'), |
---|
344 | orderno=idx) |
---|
345 | for message_elem in log_elem.children('message'): |
---|
346 | build_log.messages.append((message_elem.attr['level'], |
---|
347 | message_elem.gettext())) |
---|
348 | build_log.insert(db=db) |
---|
349 | |
---|
350 | # Collect report data from the request body |
---|
351 | for report_elem in elem.children('report'): |
---|
352 | report = Report(self.env, build=build.id, step=stepname, |
---|
353 | category=report_elem.attr.get('category'), |
---|
354 | generator=report_elem.attr.get('generator')) |
---|
355 | for item_elem in report_elem.children(): |
---|
356 | item = {'type': item_elem.name} |
---|
357 | item.update(item_elem.attr) |
---|
358 | for child_elem in item_elem.children(): |
---|
359 | item[child_elem.name] = child_elem.gettext() |
---|
360 | report.items.append(item) |
---|
361 | report.insert(db=db) |
---|
362 | |
---|
363 | # If this was the last step in the recipe we mark the build as |
---|
364 | # completed otherwise just update last_activity |
---|
365 | if last_step: |
---|
366 | self.log.info('Slave %s completed build %d ("%s" as of [%s])', |
---|
367 | build.slave, build.id, build.config, build.rev) |
---|
368 | build.stopped = step.stopped |
---|
369 | build.last_activity = build.stopped |
---|
370 | |
---|
371 | # Determine overall outcome of the build by checking the outcome |
---|
372 | # of the individual steps against the "onerror" specification of |
---|
373 | # each step in the recipe |
---|
374 | for num, recipe_step in enumerate(recipe): |
---|
375 | step = BuildStep.fetch(self.env, build.id, recipe_step.id) |
---|
376 | if step.status == BuildStep.FAILURE: |
---|
377 | if recipe_step.onerror == 'fail' or \ |
---|
378 | recipe_step.onerror == 'continue': |
---|
379 | build.status = Build.FAILURE |
---|
380 | break |
---|
381 | else: |
---|
382 | build.status = Build.SUCCESS |
---|
383 | |
---|
384 | build.update(db=db) |
---|
385 | else: |
---|
386 | build.last_activity = step.stopped |
---|
387 | build.update(db=db) |
---|
388 | |
---|
389 | # start the next step. |
---|
390 | for num, recipe_step in enumerate(recipe): |
---|
391 | if num == index + 1: |
---|
392 | next_step = recipe_step |
---|
393 | if next_step is None: |
---|
394 | self._send_error(req, HTTP_FORBIDDEN, |
---|
395 | 'Unable to find step after ' % stepname) |
---|
396 | |
---|
397 | step = self._start_new_step(build, next_step.id) |
---|
398 | step.insert(db=db) |
---|
399 | |
---|
400 | db.commit() |
---|
401 | |
---|
402 | if last_step: |
---|
403 | for listener in BuildSystem(self.env).listeners: |
---|
404 | listener.build_completed(build) |
---|
405 | |
---|
406 | body = 'Build step processed' |
---|
407 | self._send_response(req, 201, body, { |
---|
408 | 'Content-Type': 'text/plain', |
---|
409 | 'Content-Length': str(len(body)), |
---|
410 | 'Location': req.abs_href.builds( |
---|
411 | build.id, 'steps', stepname)}) |
---|
412 | |
---|
413 | def _process_attachment(self, req, config, build): |
---|
414 | resource_id = req.args['member'] == 'config' \ |
---|
415 | and build.config or build.resource.id |
---|
416 | upload = req.args['file'] |
---|
417 | if not upload.file: |
---|
418 | send_error(req, message="Attachment not received.") |
---|
419 | self.log.debug('Received attachment %s for attaching to build:%s', |
---|
420 | upload.filename, resource_id) |
---|
421 | |
---|
422 | # Determine size of file |
---|
423 | upload.file.seek(0, 2) # to the end |
---|
424 | size = upload.file.tell() |
---|
425 | upload.file.seek(0) # beginning again |
---|
426 | |
---|
427 | # Delete attachment if it already exists |
---|
428 | try: |
---|
429 | old_attach = Attachment(self.env, 'build', |
---|
430 | parent_id=resource_id, filename=upload.filename) |
---|
431 | old_attach.delete() |
---|
432 | except ResourceNotFound: |
---|
433 | pass |
---|
434 | |
---|
435 | # Save new attachment |
---|
436 | attachment = Attachment(self.env, 'build', parent_id=resource_id) |
---|
437 | attachment.description = req.args.get('description', '') |
---|
438 | attachment.author = req.authname |
---|
439 | attachment.insert(upload.filename, upload.file, size) |
---|
440 | |
---|
441 | self._send_response(req, 201, 'Attachment created', headers={ |
---|
442 | 'Content-Type': 'text/plain', |
---|
443 | 'Content-Length': str(len('Attachment created'))}) |
---|
444 | |
---|
445 | def _process_keepalive(self, req, config, build): |
---|
446 | build.last_activity = int(time.time()) |
---|
447 | build.update() |
---|
448 | |
---|
449 | self.log.info('Slave %s build %d keepalive ("%s" as of [%s])', |
---|
450 | build.slave, build.id, build.config, build.rev) |
---|
451 | |
---|
452 | body = 'Keepalive processed' |
---|
453 | self._send_response(req, 200, body, { |
---|
454 | 'Content-Type': 'text/plain', |
---|
455 | 'Content-Length': str(len(body))}) |
---|
456 | |
---|
457 | def _start_new_step(self, build, stepname): |
---|
458 | """Creates the in-memory representation for a newly started |
---|
459 | step, ready to be persisted to the database. |
---|
460 | """ |
---|
461 | step = BuildStep(self.env, build=build.id, name=stepname) |
---|
462 | step.status = BuildStep.IN_PROGRESS |
---|
463 | step.started = int(time.time()) |
---|
464 | step.stopped = 0 |
---|
465 | |
---|
466 | return step |
---|