1 | # -*- coding: utf-8 -*- |
---|
2 | # |
---|
3 | # Copyright (C) 2007-2010 Edgewall Software |
---|
4 | # Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de> |
---|
5 | # All rights reserved. |
---|
6 | # |
---|
7 | # This software is licensed as described in the file COPYING, which |
---|
8 | # you should have received as part of this distribution. The terms |
---|
9 | # are also available at http://bitten.edgewall.org/wiki/License. |
---|
10 | |
---|
11 | """Implementation of the build slave.""" |
---|
12 | |
---|
13 | from datetime import datetime |
---|
14 | import errno |
---|
15 | import urllib |
---|
16 | import urllib2 |
---|
17 | import logging |
---|
18 | import os |
---|
19 | import platform |
---|
20 | import shutil |
---|
21 | import socket |
---|
22 | import sys |
---|
23 | import tempfile |
---|
24 | import time |
---|
25 | import re |
---|
26 | import cookielib |
---|
27 | import threading |
---|
28 | import mimetools |
---|
29 | from ConfigParser import MissingSectionHeaderError |
---|
30 | |
---|
31 | from bitten import PROTOCOL_VERSION |
---|
32 | from bitten.build import BuildError |
---|
33 | from bitten.build.config import Configuration, ConfigFileNotFound |
---|
34 | from bitten.recipe import Recipe |
---|
35 | from bitten.util import xmlio |
---|
36 | from bitten.util.compat import HTTPBasicAuthHandler |
---|
37 | |
---|
38 | EX_OK = getattr(os, "EX_OK", 0) |
---|
39 | EX_UNAVAILABLE = getattr(os, "EX_UNAVAILABLE", 69) |
---|
40 | EX_IOERR = getattr(os, "EX_IOERR", 74) |
---|
41 | EX_PROTOCOL = getattr(os, "EX_PROTOCOL", 76) |
---|
42 | EX_NOPERM = getattr(os, "EX_NOPERM", 77) |
---|
43 | |
---|
44 | FORM_TOKEN_RE = re.compile('__FORM_TOKEN\" value=\"(.+)\"') |
---|
45 | |
---|
46 | __all__ = ['BuildSlave', 'ExitSlave'] |
---|
47 | __docformat__ = 'restructuredtext en' |
---|
48 | |
---|
49 | log = logging.getLogger('bitten.slave') |
---|
50 | |
---|
51 | # List of network errors which are usually temporary and non critical. |
---|
52 | temp_net_errors = [errno.ENETUNREACH, errno.ENETDOWN, errno.ETIMEDOUT, |
---|
53 | errno.ECONNREFUSED] |
---|
54 | |
---|
55 | def _rmtree(root): |
---|
56 | """Catch shutil.rmtree failures on Windows when files are read-only, and only remove if root exists.""" |
---|
57 | def _handle_error(fn, path, excinfo): |
---|
58 | os.chmod(path, 0666) |
---|
59 | fn(path) |
---|
60 | if os.path.exists(root): |
---|
61 | return shutil.rmtree(root, onerror=_handle_error) |
---|
62 | else: |
---|
63 | return False |
---|
64 | |
---|
65 | |
---|
66 | class SaneHTTPRequest(urllib2.Request): |
---|
67 | |
---|
68 | def __init__(self, method, url, data=None, headers={}): |
---|
69 | urllib2.Request.__init__(self, url, data, headers) |
---|
70 | self.method = method |
---|
71 | |
---|
72 | def get_method(self): |
---|
73 | if self.method is None: |
---|
74 | self.method = self.has_data() and 'POST' or 'GET' |
---|
75 | return self.method |
---|
76 | |
---|
77 | |
---|
78 | def encode_multipart_formdata(fields): |
---|
79 | """ |
---|
80 | Given a dictionary field parameters, returns the HTTP request body and the |
---|
81 | content_type (which includes the boundary string), to be used with an |
---|
82 | httplib-like call. |
---|
83 | |
---|
84 | Normal key/value items are treated as regular parameters, but key/tuple |
---|
85 | items are treated as files, where a value tuple is a (filename, data) tuple. |
---|
86 | |
---|
87 | For example:: |
---|
88 | |
---|
89 | fields = { |
---|
90 | 'foo': 'bar', |
---|
91 | 'foofile': ('foofile.txt', 'contents of foofile'), |
---|
92 | } |
---|
93 | body, content_type = encode_multipart_formdata(fields) |
---|
94 | |
---|
95 | Note: Adapted from http://code.google.com/p/urllib3/ (MIT license) |
---|
96 | """ |
---|
97 | |
---|
98 | BOUNDARY = mimetools.choose_boundary() |
---|
99 | ENCODE_TEMPLATE= "--%(boundary)s\r\n" \ |
---|
100 | "Content-Disposition: form-data; name=\"%(name)s\"\r\n" \ |
---|
101 | "\r\n%(value)s\r\n" |
---|
102 | ENCODE_TEMPLATE_FILE = "--%(boundary)s\r\n" \ |
---|
103 | "Content-Disposition: form-data; name=\"%(name)s\"; " \ |
---|
104 | "filename=\"%(filename)s\"\r\n" \ |
---|
105 | "Content-Type: %(contenttype)s\r\n" \ |
---|
106 | "\r\n%(value)s\r\n" |
---|
107 | |
---|
108 | body = "" |
---|
109 | for key, value in fields.iteritems(): |
---|
110 | if isinstance(value, tuple): |
---|
111 | filename, value = value |
---|
112 | body += ENCODE_TEMPLATE_FILE % { |
---|
113 | 'boundary': BOUNDARY, |
---|
114 | 'name': str(key), |
---|
115 | 'value': str(value), |
---|
116 | 'filename': str(filename), |
---|
117 | 'contenttype': 'application/octet-stream' |
---|
118 | } |
---|
119 | else: |
---|
120 | body += ENCODE_TEMPLATE % { |
---|
121 | 'boundary': BOUNDARY, |
---|
122 | 'name': str(key), |
---|
123 | 'value': str(value) |
---|
124 | } |
---|
125 | body += '--%s--\r\n' % BOUNDARY |
---|
126 | content_type = 'multipart/form-data; boundary=%s' % BOUNDARY |
---|
127 | return body, content_type |
---|
128 | |
---|
129 | |
---|
130 | class KeepAliveThread(threading.Thread): |
---|
131 | "A thread to periodically send keep-alive messages to the master" |
---|
132 | |
---|
133 | def __init__(self, opener, build_url, single_build, keepalive_interval): |
---|
134 | threading.Thread.__init__(self, None, None, "KeepaliveThread") |
---|
135 | self.build_url = build_url |
---|
136 | self.keepalive_interval = keepalive_interval |
---|
137 | self.single_build = single_build |
---|
138 | self.last_keepalive = int(time.time()) |
---|
139 | self.kill = False |
---|
140 | self.opener = opener |
---|
141 | |
---|
142 | def keepalive(self): |
---|
143 | log.debug('Sending keepalive') |
---|
144 | method = 'POST' |
---|
145 | url = self.build_url + '/keepalive/' |
---|
146 | body = None |
---|
147 | shutdown = False |
---|
148 | headers = { |
---|
149 | 'Content-Type': 'application/x-bitten+xml', |
---|
150 | 'Content-Length': '0' |
---|
151 | } |
---|
152 | log.debug('Sending %s request to %r', method, url) |
---|
153 | req = SaneHTTPRequest(method, url, body, headers or {}) |
---|
154 | try: |
---|
155 | return self.opener.open(req) |
---|
156 | except urllib2.HTTPError, e: |
---|
157 | # a conflict error lets us know that we've been |
---|
158 | # invalidated. Ideally, we'd engineer something to stop any |
---|
159 | # running steps in progress, but killing threads is tricky |
---|
160 | # stuff. For now, we'll wait for whatever's going |
---|
161 | # on to stop, and the main thread'll figure out that we've |
---|
162 | # been invalidated. |
---|
163 | log.warning('Server returned keepalive error %d: %s', e.code, e.msg) |
---|
164 | except: |
---|
165 | log.warning('Server returned unknown keepalive error') |
---|
166 | |
---|
167 | def run(self): |
---|
168 | log.debug('Keepalive thread starting.') |
---|
169 | while (not self.kill): |
---|
170 | now = int(time.time()) |
---|
171 | if (self.last_keepalive + self.keepalive_interval) < now: |
---|
172 | self.keepalive() |
---|
173 | self.last_keepalive = now |
---|
174 | |
---|
175 | time.sleep(1) |
---|
176 | log.debug('Keepalive thread exiting.') |
---|
177 | |
---|
178 | def stop(self): |
---|
179 | log.debug('Stopping keepalive thread') |
---|
180 | self.kill = True |
---|
181 | self.join(30) |
---|
182 | log.debug('Keepalive thread stopped') |
---|
183 | |
---|
184 | |
---|
185 | class BuildSlave(object): |
---|
186 | """HTTP client implementation for the build slave.""" |
---|
187 | |
---|
188 | def __init__(self, urls, name=None, config=None, dry_run=False, |
---|
189 | work_dir=None, build_dir="build_${build}", |
---|
190 | keep_files=False, single_build=False, |
---|
191 | poll_interval=300, keepalive_interval = 60, |
---|
192 | username=None, password=None, |
---|
193 | dump_reports=False, no_loop=False, form_auth=False): |
---|
194 | """Create the build slave instance. |
---|
195 | |
---|
196 | :param urls: a list of URLs of the build masters to connect to, or a |
---|
197 | single-element list containing the path to a build recipe |
---|
198 | file |
---|
199 | :param name: the name with which this slave should identify itself |
---|
200 | :param config: the path to the slave configuration file |
---|
201 | :param dry_run: wether the build outcome should not be reported back |
---|
202 | to the master |
---|
203 | :param work_dir: the working directory to use for build execution |
---|
204 | :param build_dir: the pattern to use for naming the build subdir |
---|
205 | :param keep_files: whether files and directories created for build |
---|
206 | execution should be kept when done |
---|
207 | :param single_build: whether this slave should exit after completing a |
---|
208 | single build, or continue processing builds forever |
---|
209 | :param poll_interval: the time in seconds to wait between requesting |
---|
210 | builds from the build master (default is five |
---|
211 | minutes) |
---|
212 | :param keepalive_interval: the time in seconds to wait between sending |
---|
213 | keepalive heartbeats (default is 30 seconds) |
---|
214 | :param username: the username to use when authentication against the |
---|
215 | build master is requested |
---|
216 | :param password: the password to use when authentication is needed |
---|
217 | :param dump_reports: whether report data should be written to the |
---|
218 | standard output, in addition to being transmitted |
---|
219 | to the build master |
---|
220 | :param no_loop: for this slave to just perform a single check, regardless |
---|
221 | of whether a build is done or not |
---|
222 | :param form_auth: login using AccountManager HTML form instead of |
---|
223 | HTTP authentication for all urls |
---|
224 | """ |
---|
225 | self.local = len(urls) == 1 and not urls[0].startswith('http://') \ |
---|
226 | and not urls[0].startswith('https://') |
---|
227 | if self.local: |
---|
228 | self.urls = urls |
---|
229 | else: |
---|
230 | self.urls = [ |
---|
231 | not url.endswith('/builds') and url.rstrip('/') + '/builds' |
---|
232 | or url for url in urls] |
---|
233 | |
---|
234 | if name is None: |
---|
235 | name = platform.node().split('.', 1)[0].lower() |
---|
236 | self.name = name |
---|
237 | self.config = Configuration(config) |
---|
238 | self.dry_run = dry_run |
---|
239 | if not work_dir: |
---|
240 | work_dir = tempfile.mkdtemp(prefix='bitten') |
---|
241 | elif not os.path.exists(work_dir): |
---|
242 | os.makedirs(work_dir) |
---|
243 | self.work_dir = work_dir |
---|
244 | self.build_dir = build_dir |
---|
245 | self.keep_files = keep_files |
---|
246 | self.single_build = single_build |
---|
247 | self.no_loop = no_loop |
---|
248 | self.poll_interval = poll_interval |
---|
249 | self.keepalive_interval = keepalive_interval |
---|
250 | self.dump_reports = dump_reports |
---|
251 | self.cookiejar = cookielib.CookieJar() |
---|
252 | self.username = username \ |
---|
253 | or self.config['authentication.username'] or '' |
---|
254 | |
---|
255 | if not self.local: |
---|
256 | self.password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() |
---|
257 | if self.username: |
---|
258 | log.debug('Enabling authentication with username %r', |
---|
259 | self.username) |
---|
260 | self.form_auth = form_auth |
---|
261 | password = password \ |
---|
262 | or self.config['authentication.password'] or '' |
---|
263 | self.config.packages.pop('authentication', None) |
---|
264 | urls = [url[:-len('/builds')] for url in self.urls] |
---|
265 | self.password_mgr.add_password( |
---|
266 | None, urls, self.username, password) |
---|
267 | self.auth_map = dict(map(lambda x: (x, False), urls)) |
---|
268 | |
---|
269 | def _get_opener(self): |
---|
270 | opener = urllib2.build_opener(urllib2.HTTPErrorProcessor()) |
---|
271 | opener.add_handler(HTTPBasicAuthHandler(self.password_mgr)) |
---|
272 | opener.add_handler(urllib2.HTTPDigestAuthHandler(self.password_mgr)) |
---|
273 | opener.add_handler(urllib2.HTTPCookieProcessor(self.cookiejar)) |
---|
274 | return opener |
---|
275 | opener = property(_get_opener) |
---|
276 | |
---|
277 | def request(self, method, url, body=None, headers=None): |
---|
278 | log.debug('Sending %s request to %r', method, url) |
---|
279 | req = SaneHTTPRequest(method, url, body, headers or {}) |
---|
280 | try: |
---|
281 | resp = self.opener.open(req) |
---|
282 | if not hasattr(resp, 'code'): |
---|
283 | resp.code = 200 |
---|
284 | return resp |
---|
285 | except urllib2.HTTPError, e: |
---|
286 | if e.code >= 300: |
---|
287 | if hasattr(e, 'headers') and \ |
---|
288 | e.headers.getheader('Content-Type', '' |
---|
289 | ).startswith('text/plain'): |
---|
290 | content = e.read() |
---|
291 | else: |
---|
292 | content = 'no message available' |
---|
293 | log.debug('Server returned error %d: %s (%s)', |
---|
294 | e.code, e.msg, content) |
---|
295 | raise |
---|
296 | return e |
---|
297 | |
---|
298 | def run(self): |
---|
299 | if self.local: |
---|
300 | fileobj = open(self.urls[0]) |
---|
301 | try: |
---|
302 | self._execute_build(None, fileobj) |
---|
303 | finally: |
---|
304 | fileobj.close() |
---|
305 | return EX_OK |
---|
306 | |
---|
307 | urls = [] |
---|
308 | while True: |
---|
309 | if not urls: |
---|
310 | urls[:] = self.urls |
---|
311 | url = urls.pop(0) |
---|
312 | try: |
---|
313 | try: |
---|
314 | if self.username and not self.auth_map.get(url): |
---|
315 | login_url = '%s/login?referer=%s' % (url[:-7], |
---|
316 | urllib.quote_plus(url)) |
---|
317 | # First request to url, authentication needed |
---|
318 | if self.form_auth: |
---|
319 | log.debug('Performing http form authentication') |
---|
320 | resp = self.request('POST', login_url) |
---|
321 | match = FORM_TOKEN_RE.search(resp.read()) |
---|
322 | if not match: |
---|
323 | log.error("Project %s does not support form " |
---|
324 | "authentication" % url[:-7]) |
---|
325 | raise ExitSlave(EX_NOPERM) |
---|
326 | values = {'user': self.username, |
---|
327 | 'password': |
---|
328 | self.password_mgr.find_user_password( |
---|
329 | None, url)[1], |
---|
330 | 'referer': '', |
---|
331 | '__FORM_TOKEN': match.group(1)} |
---|
332 | self.request('POST', login_url, |
---|
333 | body=urllib.urlencode(values)) |
---|
334 | else: |
---|
335 | log.debug('Performing basic/digest authentication') |
---|
336 | self.request('HEAD', login_url) |
---|
337 | self.auth_map[url] = True |
---|
338 | elif self.username: |
---|
339 | log.debug('Reusing authentication information.') |
---|
340 | else: |
---|
341 | log.debug('Authentication not provided. Attempting to ' |
---|
342 | 'execute build anonymously.') |
---|
343 | job_done = self._create_build(url) |
---|
344 | if job_done: |
---|
345 | continue |
---|
346 | except urllib2.HTTPError, e: |
---|
347 | # HTTPError doesn't have the "reason" attribute of URLError |
---|
348 | log.error(e) |
---|
349 | raise ExitSlave(EX_UNAVAILABLE) |
---|
350 | except urllib2.URLError, e: |
---|
351 | # Is this a temporary network glitch or something a bit |
---|
352 | # more severe? |
---|
353 | if isinstance(e.reason, socket.error) and \ |
---|
354 | e.reason.args[0] in temp_net_errors: |
---|
355 | log.warning(e) |
---|
356 | else: |
---|
357 | log.error(e) |
---|
358 | raise ExitSlave(EX_UNAVAILABLE) |
---|
359 | except ExitSlave, e: |
---|
360 | return e.exit_code |
---|
361 | if self.no_loop: |
---|
362 | break |
---|
363 | time.sleep(self.poll_interval) |
---|
364 | |
---|
365 | def quit(self): |
---|
366 | log.info('Shutting down') |
---|
367 | raise ExitSlave(EX_OK) |
---|
368 | |
---|
369 | def _create_build(self, url): |
---|
370 | xml = xmlio.Element('slave', name=self.name, version=PROTOCOL_VERSION)[ |
---|
371 | xmlio.Element('platform', processor=self.config['processor'])[ |
---|
372 | self.config['machine'] |
---|
373 | ], |
---|
374 | xmlio.Element('os', family=self.config['family'], |
---|
375 | version=self.config['version'])[ |
---|
376 | self.config['os'] |
---|
377 | ], |
---|
378 | ] |
---|
379 | |
---|
380 | log.debug('Configured packages: %s', self.config.packages) |
---|
381 | for package, properties in self.config.packages.items(): |
---|
382 | xml.append(xmlio.Element('package', name=package, **properties)) |
---|
383 | |
---|
384 | body = str(xml) |
---|
385 | log.debug('Sending slave configuration: %s', body) |
---|
386 | resp = self.request('POST', url, body, { |
---|
387 | 'Content-Length': str(len(body)), |
---|
388 | 'Content-Type': 'application/x-bitten+xml' |
---|
389 | }) |
---|
390 | |
---|
391 | if resp.code == 201: |
---|
392 | self._initiate_build(resp.info().get('location')) |
---|
393 | return True |
---|
394 | elif resp.code == 204: |
---|
395 | log.info('No pending builds') |
---|
396 | return False |
---|
397 | else: |
---|
398 | log.error('Unexpected response (%d %s)', resp.code, resp.msg) |
---|
399 | raise ExitSlave(EX_PROTOCOL) |
---|
400 | |
---|
401 | def _initiate_build(self, build_url): |
---|
402 | log.info('Build pending at %s', build_url) |
---|
403 | try: |
---|
404 | resp = self.request('GET', build_url) |
---|
405 | if resp.code == 200: |
---|
406 | self._execute_build(build_url, resp) |
---|
407 | else: |
---|
408 | log.error('Unexpected response (%d): %s', resp.code, resp.msg) |
---|
409 | self._cancel_build(build_url, exit_code=EX_PROTOCOL) |
---|
410 | except KeyboardInterrupt: |
---|
411 | log.warning('Build interrupted') |
---|
412 | self._cancel_build(build_url) |
---|
413 | |
---|
414 | def _execute_build(self, build_url, fileobj): |
---|
415 | build_id = build_url and int(build_url.split('/')[-1]) or 0 |
---|
416 | xml = xmlio.parse(fileobj) |
---|
417 | basedir = '' |
---|
418 | try: |
---|
419 | if not self.local: |
---|
420 | keepalive_thread = KeepAliveThread(self.opener, build_url, |
---|
421 | self.single_build, self.keepalive_interval) |
---|
422 | keepalive_thread.start() |
---|
423 | recipe = Recipe(xml, os.path.join(self.work_dir, self.build_dir), |
---|
424 | self.config) |
---|
425 | basedir = recipe.ctxt.basedir |
---|
426 | log.debug('Running build in directory %s' % basedir) |
---|
427 | if not os.path.exists(basedir): |
---|
428 | os.mkdir(basedir) |
---|
429 | |
---|
430 | for step in recipe: |
---|
431 | try: |
---|
432 | log.info('Executing build step %r, onerror = %s', step.id, step.onerror) |
---|
433 | if not self._execute_step(build_url, recipe, step): |
---|
434 | log.warning('Stopping build due to failure') |
---|
435 | break |
---|
436 | except Exception, e: |
---|
437 | log.error('Exception raised processing step %s. Reraising %s', step.id, e) |
---|
438 | raise |
---|
439 | else: |
---|
440 | log.info('Build completed') |
---|
441 | if self.dry_run: |
---|
442 | self._cancel_build(build_url) |
---|
443 | finally: |
---|
444 | if not self.local: |
---|
445 | keepalive_thread.stop() |
---|
446 | if not self.keep_files and os.path.isdir(basedir): |
---|
447 | log.debug('Removing build directory %s' % basedir) |
---|
448 | _rmtree(basedir) |
---|
449 | if self.single_build: |
---|
450 | log.info('Exiting after single build completed.') |
---|
451 | raise ExitSlave(EX_OK) |
---|
452 | |
---|
453 | def _execute_step(self, build_url, recipe, step): |
---|
454 | failed = False |
---|
455 | started = int(time.time()) |
---|
456 | xml = xmlio.Element('result', step=step.id) |
---|
457 | try: |
---|
458 | for type, category, generator, output in \ |
---|
459 | step.execute(recipe.ctxt): |
---|
460 | if type == Recipe.ERROR: |
---|
461 | failed = True |
---|
462 | if type == Recipe.REPORT and self.dump_reports: |
---|
463 | print output |
---|
464 | if type == Recipe.ATTACH: |
---|
465 | # Attachments are added out-of-band due to major |
---|
466 | # performance issues with inlined base64 xml content |
---|
467 | self._attach_file(build_url, recipe, output) |
---|
468 | xml.append(xmlio.Element(type, category=category, |
---|
469 | generator=generator)[ |
---|
470 | output |
---|
471 | ]) |
---|
472 | except KeyboardInterrupt: |
---|
473 | log.warning('Build interrupted') |
---|
474 | self._cancel_build(build_url) |
---|
475 | except BuildError, e: |
---|
476 | log.error('Build step %r failed', step.id) |
---|
477 | failed = True |
---|
478 | except Exception, e: |
---|
479 | log.error('Internal error in build step %r', step.id, exc_info=True) |
---|
480 | failed = True |
---|
481 | xml.attr['duration'] = (time.time() - started) |
---|
482 | if failed: |
---|
483 | xml.attr['status'] = 'failure' |
---|
484 | else: |
---|
485 | xml.attr['status'] = 'success' |
---|
486 | log.info('Build step %s completed successfully', step.id) |
---|
487 | |
---|
488 | if not self.local and not self.dry_run: |
---|
489 | try: |
---|
490 | resp = self.request('POST', build_url + '/steps/', str(xml), { |
---|
491 | 'Content-Type': 'application/x-bitten+xml' |
---|
492 | }) |
---|
493 | if resp.code != 201: |
---|
494 | log.error('Unexpected response (%d): %s', resp.code, |
---|
495 | resp.msg) |
---|
496 | except KeyboardInterrupt: |
---|
497 | log.warning('Build interrupted') |
---|
498 | self._cancel_build(build_url) |
---|
499 | return not failed or step.onerror != 'fail' |
---|
500 | |
---|
501 | def _cancel_build(self, build_url, exit_code=EX_OK): |
---|
502 | log.info('Cancelling build at %s', build_url) |
---|
503 | if not self.local: |
---|
504 | resp = self.request('DELETE', build_url) |
---|
505 | if resp.code not in (200, 204): |
---|
506 | log.error('Unexpected response (%d): %s', resp.code, resp.msg) |
---|
507 | raise ExitSlave(exit_code) |
---|
508 | |
---|
509 | def _attach_file(self, build_url, recipe, attachment): |
---|
510 | form_token = recipe._root.attr.get('form_token', '') |
---|
511 | if self.local or self.dry_run or not form_token: |
---|
512 | log.info('Attachment %s not sent due to current slave options', |
---|
513 | attachment.attr['filename']) |
---|
514 | return |
---|
515 | resource_type = attachment.attr['resource'] |
---|
516 | url = str(build_url + '/attach/' + resource_type) |
---|
517 | path = recipe.ctxt.resolve(attachment.attr['filename']) |
---|
518 | filename = os.path.basename(path) |
---|
519 | log.debug('Attaching file %s to %s...', attachment.attr['filename'], |
---|
520 | resource_type) |
---|
521 | f = open(path, 'rb') |
---|
522 | try: |
---|
523 | data, content_type = encode_multipart_formdata({ |
---|
524 | 'file': (filename, f.read()), |
---|
525 | 'description': attachment.attr['description'], |
---|
526 | '__FORM_TOKEN': form_token}) |
---|
527 | finally: |
---|
528 | f.close() |
---|
529 | resp = self.request('POST', url , data, { |
---|
530 | 'Content-Type': content_type}) |
---|
531 | if not resp.code == 201: |
---|
532 | msg = 'Error attaching %s to %s' |
---|
533 | log.error(msg, filename, resource_type) |
---|
534 | raise BuildError(msg, filename, resource_type) |
---|
535 | |
---|
536 | class ExitSlave(Exception): |
---|
537 | """Exception used internally by the slave to signal that the slave process |
---|
538 | should be stopped. |
---|
539 | """ |
---|
540 | def __init__(self, exit_code): |
---|
541 | self.exit_code = exit_code |
---|
542 | Exception.__init__(self) |
---|
543 | |
---|
544 | |
---|
545 | def main(): |
---|
546 | """Main entry point for running the build slave.""" |
---|
547 | from bitten import __version__ as VERSION |
---|
548 | from optparse import OptionParser |
---|
549 | |
---|
550 | parser = OptionParser(usage='usage: %prog [options] url1 [url2] ...', |
---|
551 | version='%%prog %s' % VERSION) |
---|
552 | parser.add_option('--name', action='store', dest='name', |
---|
553 | help='name of this slave (defaults to host name)') |
---|
554 | parser.add_option('-f', '--config', action='store', dest='config', |
---|
555 | metavar='FILE', help='path to configuration file') |
---|
556 | parser.add_option('-u', '--user', dest='username', |
---|
557 | help='the username to use for authentication') |
---|
558 | parser.add_option('-p', '--password', dest='password', |
---|
559 | help='the password to use when authenticating') |
---|
560 | def _ask_password(option, opt_str, value, parser): |
---|
561 | from getpass import getpass |
---|
562 | parser.values.password = getpass('Password: ') |
---|
563 | parser.add_option('-P', '--ask-password', action='callback', |
---|
564 | callback=_ask_password, help='Prompt for password') |
---|
565 | parser.add_option('--form-auth', action='store_true', |
---|
566 | dest='form_auth', |
---|
567 | help='login using AccountManager HTML form instead of ' |
---|
568 | 'HTTP authentication for all urls') |
---|
569 | |
---|
570 | group = parser.add_option_group('building') |
---|
571 | group.add_option('-d', '--work-dir', action='store', dest='work_dir', |
---|
572 | metavar='DIR', help='working directory for builds') |
---|
573 | group.add_option('--build-dir', action='store', dest='build_dir', |
---|
574 | default = 'build_${config}_${build}', |
---|
575 | help='name pattern for the build dir to use inside the ' |
---|
576 | 'working dir ["%default"]') |
---|
577 | group.add_option('-k', '--keep-files', action='store_true', |
---|
578 | dest='keep_files', |
---|
579 | help='don\'t delete files after builds') |
---|
580 | group.add_option('-s', '--single', action='store_true', |
---|
581 | dest='single_build', |
---|
582 | help='exit after completing a single build') |
---|
583 | group.add_option('', '--no-loop', action='store_true', |
---|
584 | dest='no_loop', |
---|
585 | help='exit after completing a single check and running ' |
---|
586 | 'the required builds') |
---|
587 | group.add_option('-n', '--dry-run', action='store_true', dest='dry_run', |
---|
588 | help='don\'t report results back to master') |
---|
589 | group.add_option('-i', '--interval', dest='interval', metavar='SECONDS', |
---|
590 | type='int', help='time to wait between requesting builds') |
---|
591 | group.add_option('-b', '--keepalive_interval', dest='keepalive_interval', metavar='SECONDS', type='int', help='time to wait between keepalive heartbeats') |
---|
592 | group = parser.add_option_group('logging') |
---|
593 | group.add_option('-l', '--log', dest='logfile', metavar='FILENAME', |
---|
594 | help='write log messages to FILENAME') |
---|
595 | group.add_option('-v', '--verbose', action='store_const', dest='loglevel', |
---|
596 | const=logging.DEBUG, help='print as much as possible') |
---|
597 | group.add_option('-q', '--quiet', action='store_const', dest='loglevel', |
---|
598 | const=logging.WARN, help='print as little as possible') |
---|
599 | group.add_option('--dump-reports', action='store_true', dest='dump_reports', |
---|
600 | help='whether report data should be printed') |
---|
601 | |
---|
602 | parser.set_defaults(dry_run=False, keep_files=False, |
---|
603 | loglevel=logging.INFO, single_build=False, no_loop=False, |
---|
604 | dump_reports=False, interval=300, keepalive_interval=60, |
---|
605 | form_auth=False) |
---|
606 | options, args = parser.parse_args() |
---|
607 | |
---|
608 | if len(args) < 1: |
---|
609 | parser.error('incorrect number of arguments') |
---|
610 | urls = args |
---|
611 | |
---|
612 | logger = logging.getLogger('bitten') |
---|
613 | logger.setLevel(options.loglevel) |
---|
614 | handler = logging.StreamHandler() |
---|
615 | handler.setLevel(options.loglevel) |
---|
616 | formatter = logging.Formatter('[%(levelname)-8s] %(message)s') |
---|
617 | handler.setFormatter(formatter) |
---|
618 | logger.addHandler(handler) |
---|
619 | if options.logfile: |
---|
620 | handler = logging.FileHandler(options.logfile) |
---|
621 | handler.setLevel(options.loglevel) |
---|
622 | formatter = logging.Formatter('%(asctime)s [%(name)s] %(levelname)s: ' |
---|
623 | '%(message)s') |
---|
624 | handler.setFormatter(formatter) |
---|
625 | logger.addHandler(handler) |
---|
626 | |
---|
627 | log.info("Slave launched at %s" % \ |
---|
628 | datetime.now().strftime('%Y-%m-%d %H:%M:%S')) |
---|
629 | |
---|
630 | slave = None |
---|
631 | try: |
---|
632 | slave = BuildSlave(urls, name=options.name, config=options.config, |
---|
633 | dry_run=options.dry_run, work_dir=options.work_dir, |
---|
634 | build_dir=options.build_dir, |
---|
635 | keep_files=options.keep_files, |
---|
636 | single_build=options.single_build, |
---|
637 | no_loop=options.no_loop, |
---|
638 | poll_interval=options.interval, |
---|
639 | keepalive_interval=options.keepalive_interval, |
---|
640 | username=options.username, password=options.password, |
---|
641 | dump_reports=options.dump_reports, |
---|
642 | form_auth=options.form_auth) |
---|
643 | try: |
---|
644 | exit_code = slave.run() |
---|
645 | except KeyboardInterrupt: |
---|
646 | slave.quit() |
---|
647 | except ConfigFileNotFound, e: |
---|
648 | log.error(e) |
---|
649 | exit_code = EX_IOERR |
---|
650 | except MissingSectionHeaderError: |
---|
651 | log.error("Error parsing configuration file %r. Wrong format?" \ |
---|
652 | % options.config) |
---|
653 | exit_code = EX_IOERR |
---|
654 | except ExitSlave, e: |
---|
655 | exit_code = e.exit_code |
---|
656 | |
---|
657 | if slave and not (options.work_dir or options.keep_files): |
---|
658 | log.debug('Removing working directory %s' % slave.work_dir) |
---|
659 | _rmtree(slave.work_dir) |
---|
660 | |
---|
661 | log.info("Slave exited at %s" % \ |
---|
662 | datetime.now().strftime('%Y-%m-%d %H:%M:%S')) |
---|
663 | |
---|
664 | return exit_code |
---|
665 | |
---|
666 | if __name__ == '__main__': |
---|
667 | sys.exit(main()) |
---|