| 1 | # -*- coding: utf-8 -*- |
---|
| 2 | # |
---|
| 3 | # Copyright (C) 2007-2010 Edgewall Software |
---|
| 4 | # Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de> |
---|
| 5 | # All rights reserved. |
---|
| 6 | # |
---|
| 7 | # This software is licensed as described in the file COPYING, which |
---|
| 8 | # you should have received as part of this distribution. The terms |
---|
| 9 | # are also available at http://bitten.edgewall.org/wiki/License. |
---|
| 10 | |
---|
1 | 11 | """Implementation of the build slave.""" |
---|
| 12 | |
---|
1 | 13 | from datetime import datetime |
---|
1 | 14 | import errno |
---|
1 | 15 | import urllib |
---|
1 | 16 | import urllib2 |
---|
1 | 17 | import logging |
---|
1 | 18 | import os |
---|
1 | 19 | import platform |
---|
1 | 20 | import shutil |
---|
1 | 21 | import socket |
---|
1 | 22 | import sys |
---|
1 | 23 | import tempfile |
---|
1 | 24 | import time |
---|
1 | 25 | import re |
---|
1 | 26 | import cookielib |
---|
1 | 27 | import threading |
---|
1 | 28 | import mimetools |
---|
1 | 29 | from ConfigParser import MissingSectionHeaderError |
---|
| 30 | |
---|
1 | 31 | from bitten import PROTOCOL_VERSION |
---|
1 | 32 | from bitten.build import BuildError |
---|
1 | 33 | from bitten.build.config import Configuration, ConfigFileNotFound |
---|
1 | 34 | from bitten.recipe import Recipe |
---|
1 | 35 | from bitten.util import xmlio |
---|
1 | 36 | from bitten.util.compat import HTTPBasicAuthHandler |
---|
| 37 | |
---|
1 | 38 | EX_OK = getattr(os, "EX_OK", 0) |
---|
1 | 39 | EX_UNAVAILABLE = getattr(os, "EX_UNAVAILABLE", 69) |
---|
1 | 40 | EX_IOERR = getattr(os, "EX_IOERR", 74) |
---|
1 | 41 | EX_PROTOCOL = getattr(os, "EX_PROTOCOL", 76) |
---|
1 | 42 | EX_NOPERM = getattr(os, "EX_NOPERM", 77) |
---|
| 43 | |
---|
1 | 44 | FORM_TOKEN_RE = re.compile('__FORM_TOKEN\" value=\"(.+)\"') |
---|
| 45 | |
---|
1 | 46 | __all__ = ['BuildSlave', 'ExitSlave'] |
---|
1 | 47 | __docformat__ = 'restructuredtext en' |
---|
| 48 | |
---|
1 | 49 | log = logging.getLogger('bitten.slave') |
---|
| 50 | |
---|
| 51 | # List of network errors which are usually temporary and non critical. |
---|
1 | 52 | temp_net_errors = [errno.ENETUNREACH, errno.ENETDOWN, errno.ETIMEDOUT, |
---|
1 | 53 | errno.ECONNREFUSED] |
---|
| 54 | |
---|
1 | 55 | def _rmtree(root): |
---|
| 56 | """Catch shutil.rmtree failures on Windows when files are read-only, and only remove if root exists.""" |
---|
2 | 57 | def _handle_error(fn, path, excinfo): |
---|
0 | 58 | os.chmod(path, 0666) |
---|
0 | 59 | fn(path) |
---|
2 | 60 | if os.path.exists(root): |
---|
2 | 61 | return shutil.rmtree(root, onerror=_handle_error) |
---|
2 | 62 | else: |
---|
0 | 63 | return False |
---|
| 64 | |
---|
| 65 | |
---|
2 | 66 | class SaneHTTPRequest(urllib2.Request): |
---|
| 67 | |
---|
1 | 68 | def __init__(self, method, url, data=None, headers={}): |
---|
0 | 69 | urllib2.Request.__init__(self, url, data, headers) |
---|
0 | 70 | self.method = method |
---|
| 71 | |
---|
1 | 72 | def get_method(self): |
---|
0 | 73 | if self.method is None: |
---|
0 | 74 | self.method = self.has_data() and 'POST' or 'GET' |
---|
0 | 75 | return self.method |
---|
| 76 | |
---|
| 77 | |
---|
1 | 78 | def encode_multipart_formdata(fields): |
---|
| 79 | """ |
---|
| 80 | Given a dictionary field parameters, returns the HTTP request body and the |
---|
| 81 | content_type (which includes the boundary string), to be used with an |
---|
| 82 | httplib-like call. |
---|
| 83 | |
---|
| 84 | Normal key/value items are treated as regular parameters, but key/tuple |
---|
| 85 | items are treated as files, where a value tuple is a (filename, data) tuple. |
---|
| 86 | |
---|
| 87 | For example:: |
---|
| 88 | |
---|
| 89 | fields = { |
---|
| 90 | 'foo': 'bar', |
---|
| 91 | 'foofile': ('foofile.txt', 'contents of foofile'), |
---|
| 92 | } |
---|
| 93 | body, content_type = encode_multipart_formdata(fields) |
---|
| 94 | |
---|
| 95 | Note: Adapted from http://code.google.com/p/urllib3/ (MIT license) |
---|
| 96 | """ |
---|
| 97 | |
---|
3 | 98 | BOUNDARY = mimetools.choose_boundary() |
---|
3 | 99 | ENCODE_TEMPLATE= "--%(boundary)s\r\n" \ |
---|
3 | 100 | "Content-Disposition: form-data; name=\"%(name)s\"\r\n" \ |
---|
3 | 101 | "\r\n%(value)s\r\n" |
---|
3 | 102 | ENCODE_TEMPLATE_FILE = "--%(boundary)s\r\n" \ |
---|
3 | 103 | "Content-Disposition: form-data; name=\"%(name)s\"; " \ |
---|
3 | 104 | "filename=\"%(filename)s\"\r\n" \ |
---|
3 | 105 | "Content-Type: %(contenttype)s\r\n" \ |
---|
3 | 106 | "\r\n%(value)s\r\n" |
---|
| 107 | |
---|
3 | 108 | body = "" |
---|
11 | 109 | for key, value in fields.iteritems(): |
---|
8 | 110 | if isinstance(value, tuple): |
---|
3 | 111 | filename, value = value |
---|
3 | 112 | body += ENCODE_TEMPLATE_FILE % { |
---|
3 | 113 | 'boundary': BOUNDARY, |
---|
3 | 114 | 'name': str(key), |
---|
3 | 115 | 'value': str(value), |
---|
3 | 116 | 'filename': str(filename), |
---|
3 | 117 | 'contenttype': 'application/octet-stream' |
---|
3 | 118 | } |
---|
3 | 119 | else: |
---|
5 | 120 | body += ENCODE_TEMPLATE % { |
---|
5 | 121 | 'boundary': BOUNDARY, |
---|
5 | 122 | 'name': str(key), |
---|
5 | 123 | 'value': str(value) |
---|
5 | 124 | } |
---|
3 | 125 | body += '--%s--\r\n' % BOUNDARY |
---|
3 | 126 | content_type = 'multipart/form-data; boundary=%s' % BOUNDARY |
---|
3 | 127 | return body, content_type |
---|
| 128 | |
---|
| 129 | |
---|
2 | 130 | class KeepAliveThread(threading.Thread): |
---|
1 | 131 | "A thread to periodically send keep-alive messages to the master" |
---|
| 132 | |
---|
1 | 133 | def __init__(self, opener, build_url, single_build, keepalive_interval): |
---|
0 | 134 | threading.Thread.__init__(self, None, None, "KeepaliveThread") |
---|
0 | 135 | self.build_url = build_url |
---|
0 | 136 | self.keepalive_interval = keepalive_interval |
---|
0 | 137 | self.single_build = single_build |
---|
0 | 138 | self.last_keepalive = int(time.time()) |
---|
0 | 139 | self.kill = False |
---|
0 | 140 | self.opener = opener |
---|
| 141 | |
---|
1 | 142 | def keepalive(self): |
---|
0 | 143 | log.debug('Sending keepalive') |
---|
0 | 144 | method = 'POST' |
---|
0 | 145 | url = self.build_url + '/keepalive/' |
---|
0 | 146 | body = None |
---|
0 | 147 | shutdown = False |
---|
0 | 148 | headers = { |
---|
0 | 149 | 'Content-Type': 'application/x-bitten+xml', |
---|
0 | 150 | 'Content-Length': '0' |
---|
0 | 151 | } |
---|
0 | 152 | log.debug('Sending %s request to %r', method, url) |
---|
0 | 153 | req = SaneHTTPRequest(method, url, body, headers or {}) |
---|
0 | 154 | try: |
---|
0 | 155 | return self.opener.open(req) |
---|
0 | 156 | except urllib2.HTTPError, e: |
---|
| 157 | # a conflict error lets us know that we've been |
---|
| 158 | # invalidated. Ideally, we'd engineer something to stop any |
---|
| 159 | # running steps in progress, but killing threads is tricky |
---|
| 160 | # stuff. For now, we'll wait for whatever's going |
---|
| 161 | # on to stop, and the main thread'll figure out that we've |
---|
| 162 | # been invalidated. |
---|
0 | 163 | log.warning('Server returned keepalive error %d: %s', e.code, e.msg) |
---|
0 | 164 | except: |
---|
0 | 165 | log.warning('Server returned unknown keepalive error') |
---|
| 166 | |
---|
1 | 167 | def run(self): |
---|
0 | 168 | log.debug('Keepalive thread starting.') |
---|
0 | 169 | while (not self.kill): |
---|
0 | 170 | now = int(time.time()) |
---|
0 | 171 | if (self.last_keepalive + self.keepalive_interval) < now: |
---|
0 | 172 | self.keepalive() |
---|
0 | 173 | self.last_keepalive = now |
---|
| 174 | |
---|
0 | 175 | time.sleep(1) |
---|
0 | 176 | log.debug('Keepalive thread exiting.') |
---|
| 177 | |
---|
1 | 178 | def stop(self): |
---|
0 | 179 | log.debug('Stopping keepalive thread') |
---|
0 | 180 | self.kill = True |
---|
0 | 181 | self.join(30) |
---|
0 | 182 | log.debug('Keepalive thread stopped') |
---|
| 183 | |
---|
| 184 | |
---|
2 | 185 | class BuildSlave(object): |
---|
1 | 186 | """HTTP client implementation for the build slave.""" |
---|
| 187 | |
---|
1 | 188 | def __init__(self, urls, name=None, config=None, dry_run=False, |
---|
1 | 189 | work_dir=None, build_dir="build_${build}", |
---|
1 | 190 | keep_files=False, single_build=False, |
---|
1 | 191 | poll_interval=300, keepalive_interval = 60, |
---|
1 | 192 | username=None, password=None, |
---|
1 | 193 | dump_reports=False, no_loop=False, form_auth=False): |
---|
| 194 | """Create the build slave instance. |
---|
| 195 | |
---|
| 196 | :param urls: a list of URLs of the build masters to connect to, or a |
---|
| 197 | single-element list containing the path to a build recipe |
---|
| 198 | file |
---|
| 199 | :param name: the name with which this slave should identify itself |
---|
| 200 | :param config: the path to the slave configuration file |
---|
| 201 | :param dry_run: wether the build outcome should not be reported back |
---|
| 202 | to the master |
---|
| 203 | :param work_dir: the working directory to use for build execution |
---|
| 204 | :param build_dir: the pattern to use for naming the build subdir |
---|
| 205 | :param keep_files: whether files and directories created for build |
---|
| 206 | execution should be kept when done |
---|
| 207 | :param single_build: whether this slave should exit after completing a |
---|
| 208 | single build, or continue processing builds forever |
---|
| 209 | :param poll_interval: the time in seconds to wait between requesting |
---|
| 210 | builds from the build master (default is five |
---|
| 211 | minutes) |
---|
| 212 | :param keepalive_interval: the time in seconds to wait between sending |
---|
| 213 | keepalive heartbeats (default is 30 seconds) |
---|
| 214 | :param username: the username to use when authentication against the |
---|
| 215 | build master is requested |
---|
| 216 | :param password: the password to use when authentication is needed |
---|
| 217 | :param dump_reports: whether report data should be written to the |
---|
| 218 | standard output, in addition to being transmitted |
---|
| 219 | to the build master |
---|
| 220 | :param no_loop: for this slave to just perform a single check, regardless |
---|
| 221 | of whether a build is done or not |
---|
| 222 | :param form_auth: login using AccountManager HTML form instead of |
---|
| 223 | HTTP authentication for all urls |
---|
| 224 | """ |
---|
3 | 225 | self.local = len(urls) == 1 and not urls[0].startswith('http://') \ |
---|
2 | 226 | and not urls[0].startswith('https://') |
---|
3 | 227 | if self.local: |
---|
2 | 228 | self.urls = urls |
---|
2 | 229 | else: |
---|
2 | 230 | self.urls = [ |
---|
1 | 231 | not url.endswith('/builds') and url.rstrip('/') + '/builds' |
---|
1 | 232 | or url for url in urls] |
---|
| 233 | |
---|
3 | 234 | if name is None: |
---|
3 | 235 | name = platform.node().split('.', 1)[0].lower() |
---|
3 | 236 | self.name = name |
---|
3 | 237 | self.config = Configuration(config) |
---|
3 | 238 | self.dry_run = dry_run |
---|
3 | 239 | if not work_dir: |
---|
0 | 240 | work_dir = tempfile.mkdtemp(prefix='bitten') |
---|
3 | 241 | elif not os.path.exists(work_dir): |
---|
0 | 242 | os.makedirs(work_dir) |
---|
3 | 243 | self.work_dir = work_dir |
---|
3 | 244 | self.build_dir = build_dir |
---|
3 | 245 | self.keep_files = keep_files |
---|
3 | 246 | self.single_build = single_build |
---|
3 | 247 | self.no_loop = no_loop |
---|
3 | 248 | self.poll_interval = poll_interval |
---|
3 | 249 | self.keepalive_interval = keepalive_interval |
---|
3 | 250 | self.dump_reports = dump_reports |
---|
3 | 251 | self.cookiejar = cookielib.CookieJar() |
---|
3 | 252 | self.username = username \ |
---|
3 | 253 | or self.config['authentication.username'] or '' |
---|
| 254 | |
---|
3 | 255 | if not self.local: |
---|
1 | 256 | self.password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() |
---|
1 | 257 | if self.username: |
---|
0 | 258 | log.debug('Enabling authentication with username %r', |
---|
0 | 259 | self.username) |
---|
0 | 260 | self.form_auth = form_auth |
---|
0 | 261 | password = password \ |
---|
0 | 262 | or self.config['authentication.password'] or '' |
---|
0 | 263 | self.config.packages.pop('authentication', None) |
---|
0 | 264 | urls = [url[:-len('/builds')] for url in self.urls] |
---|
0 | 265 | self.password_mgr.add_password( |
---|
0 | 266 | None, urls, self.username, password) |
---|
0 | 267 | self.auth_map = dict(map(lambda x: (x, False), urls)) |
---|
| 268 | |
---|
1 | 269 | def _get_opener(self): |
---|
0 | 270 | opener = urllib2.build_opener(urllib2.HTTPErrorProcessor()) |
---|
0 | 271 | opener.add_handler(HTTPBasicAuthHandler(self.password_mgr)) |
---|
0 | 272 | opener.add_handler(urllib2.HTTPDigestAuthHandler(self.password_mgr)) |
---|
0 | 273 | opener.add_handler(urllib2.HTTPCookieProcessor(self.cookiejar)) |
---|
0 | 274 | return opener |
---|
1 | 275 | opener = property(_get_opener) |
---|
| 276 | |
---|
1 | 277 | def request(self, method, url, body=None, headers=None): |
---|
0 | 278 | log.debug('Sending %s request to %r', method, url) |
---|
0 | 279 | req = SaneHTTPRequest(method, url, body, headers or {}) |
---|
0 | 280 | try: |
---|
0 | 281 | resp = self.opener.open(req) |
---|
0 | 282 | if not hasattr(resp, 'code'): |
---|
0 | 283 | resp.code = 200 |
---|
0 | 284 | return resp |
---|
0 | 285 | except urllib2.HTTPError, e: |
---|
0 | 286 | if e.code >= 300: |
---|
0 | 287 | if hasattr(e, 'headers') and \ |
---|
0 | 288 | e.headers.getheader('Content-Type', '' |
---|
0 | 289 | ).startswith('text/plain'): |
---|
0 | 290 | content = e.read() |
---|
0 | 291 | else: |
---|
0 | 292 | content = 'no message available' |
---|
0 | 293 | log.debug('Server returned error %d: %s (%s)', |
---|
0 | 294 | e.code, e.msg, content) |
---|
0 | 295 | raise |
---|
0 | 296 | return e |
---|
| 297 | |
---|
1 | 298 | def run(self): |
---|
2 | 299 | if self.local: |
---|
2 | 300 | fileobj = open(self.urls[0]) |
---|
2 | 301 | try: |
---|
2 | 302 | self._execute_build(None, fileobj) |
---|
2 | 303 | finally: |
---|
2 | 304 | fileobj.close() |
---|
2 | 305 | return EX_OK |
---|
| 306 | |
---|
0 | 307 | urls = [] |
---|
0 | 308 | while True: |
---|
0 | 309 | if not urls: |
---|
0 | 310 | urls[:] = self.urls |
---|
0 | 311 | url = urls.pop(0) |
---|
0 | 312 | try: |
---|
0 | 313 | try: |
---|
0 | 314 | if self.username and not self.auth_map.get(url): |
---|
0 | 315 | login_url = '%s/login?referer=%s' % (url[:-7], |
---|
0 | 316 | urllib.quote_plus(url)) |
---|
| 317 | # First request to url, authentication needed |
---|
0 | 318 | if self.form_auth: |
---|
0 | 319 | log.debug('Performing http form authentication') |
---|
0 | 320 | resp = self.request('POST', login_url) |
---|
0 | 321 | match = FORM_TOKEN_RE.search(resp.read()) |
---|
0 | 322 | if not match: |
---|
0 | 323 | log.error("Project %s does not support form " |
---|
0 | 324 | "authentication" % url[:-7]) |
---|
0 | 325 | raise ExitSlave(EX_NOPERM) |
---|
0 | 326 | values = {'user': self.username, |
---|
0 | 327 | 'password': |
---|
0 | 328 | self.password_mgr.find_user_password( |
---|
0 | 329 | None, url)[1], |
---|
0 | 330 | 'referer': '', |
---|
0 | 331 | '__FORM_TOKEN': match.group(1)} |
---|
0 | 332 | self.request('POST', login_url, |
---|
0 | 333 | body=urllib.urlencode(values)) |
---|
0 | 334 | else: |
---|
0 | 335 | log.debug('Performing basic/digest authentication') |
---|
0 | 336 | self.request('HEAD', login_url) |
---|
0 | 337 | self.auth_map[url] = True |
---|
0 | 338 | elif self.username: |
---|
0 | 339 | log.debug('Reusing authentication information.') |
---|
0 | 340 | else: |
---|
0 | 341 | log.debug('Authentication not provided. Attempting to ' |
---|
0 | 342 | 'execute build anonymously.') |
---|
0 | 343 | job_done = self._create_build(url) |
---|
0 | 344 | if job_done: |
---|
0 | 345 | continue |
---|
0 | 346 | except urllib2.HTTPError, e: |
---|
| 347 | # HTTPError doesn't have the "reason" attribute of URLError |
---|
0 | 348 | log.error(e) |
---|
0 | 349 | raise ExitSlave(EX_UNAVAILABLE) |
---|
0 | 350 | except urllib2.URLError, e: |
---|
| 351 | # Is this a temporary network glitch or something a bit |
---|
| 352 | # more severe? |
---|
0 | 353 | if isinstance(e.reason, socket.error) and \ |
---|
0 | 354 | e.reason.args[0] in temp_net_errors: |
---|
0 | 355 | log.warning(e) |
---|
0 | 356 | else: |
---|
0 | 357 | log.error(e) |
---|
0 | 358 | raise ExitSlave(EX_UNAVAILABLE) |
---|
0 | 359 | except ExitSlave, e: |
---|
0 | 360 | return e.exit_code |
---|
0 | 361 | if self.no_loop: |
---|
0 | 362 | break |
---|
0 | 363 | time.sleep(self.poll_interval) |
---|
| 364 | |
---|
1 | 365 | def quit(self): |
---|
1 | 366 | log.info('Shutting down') |
---|
1 | 367 | raise ExitSlave(EX_OK) |
---|
| 368 | |
---|
1 | 369 | def _create_build(self, url): |
---|
0 | 370 | xml = xmlio.Element('slave', name=self.name, version=PROTOCOL_VERSION)[ |
---|
0 | 371 | xmlio.Element('platform', processor=self.config['processor'])[ |
---|
0 | 372 | self.config['machine'] |
---|
0 | 373 | ], |
---|
0 | 374 | xmlio.Element('os', family=self.config['family'], |
---|
0 | 375 | version=self.config['version'])[ |
---|
0 | 376 | self.config['os'] |
---|
0 | 377 | ], |
---|
0 | 378 | ] |
---|
| 379 | |
---|
0 | 380 | log.debug('Configured packages: %s', self.config.packages) |
---|
0 | 381 | for package, properties in self.config.packages.items(): |
---|
0 | 382 | xml.append(xmlio.Element('package', name=package, **properties)) |
---|
| 383 | |
---|
0 | 384 | body = str(xml) |
---|
0 | 385 | log.debug('Sending slave configuration: %s', body) |
---|
0 | 386 | resp = self.request('POST', url, body, { |
---|
0 | 387 | 'Content-Length': str(len(body)), |
---|
0 | 388 | 'Content-Type': 'application/x-bitten+xml' |
---|
0 | 389 | }) |
---|
| 390 | |
---|
0 | 391 | if resp.code == 201: |
---|
0 | 392 | self._initiate_build(resp.info().get('location')) |
---|
0 | 393 | return True |
---|
0 | 394 | elif resp.code == 204: |
---|
0 | 395 | log.info('No pending builds') |
---|
0 | 396 | return False |
---|
0 | 397 | else: |
---|
0 | 398 | log.error('Unexpected response (%d %s)', resp.code, resp.msg) |
---|
0 | 399 | raise ExitSlave(EX_PROTOCOL) |
---|
| 400 | |
---|
1 | 401 | def _initiate_build(self, build_url): |
---|
0 | 402 | log.info('Build pending at %s', build_url) |
---|
0 | 403 | try: |
---|
0 | 404 | resp = self.request('GET', build_url) |
---|
0 | 405 | if resp.code == 200: |
---|
0 | 406 | self._execute_build(build_url, resp) |
---|
0 | 407 | else: |
---|
0 | 408 | log.error('Unexpected response (%d): %s', resp.code, resp.msg) |
---|
0 | 409 | self._cancel_build(build_url, exit_code=EX_PROTOCOL) |
---|
0 | 410 | except KeyboardInterrupt: |
---|
0 | 411 | log.warning('Build interrupted') |
---|
0 | 412 | self._cancel_build(build_url) |
---|
| 413 | |
---|
1 | 414 | def _execute_build(self, build_url, fileobj): |
---|
2 | 415 | build_id = build_url and int(build_url.split('/')[-1]) or 0 |
---|
2 | 416 | xml = xmlio.parse(fileobj) |
---|
2 | 417 | basedir = '' |
---|
2 | 418 | try: |
---|
2 | 419 | if not self.local: |
---|
0 | 420 | keepalive_thread = KeepAliveThread(self.opener, build_url, |
---|
0 | 421 | self.single_build, self.keepalive_interval) |
---|
0 | 422 | keepalive_thread.start() |
---|
2 | 423 | recipe = Recipe(xml, os.path.join(self.work_dir, self.build_dir), |
---|
2 | 424 | self.config) |
---|
2 | 425 | basedir = recipe.ctxt.basedir |
---|
2 | 426 | log.debug('Running build in directory %s' % basedir) |
---|
2 | 427 | if not os.path.exists(basedir): |
---|
2 | 428 | os.mkdir(basedir) |
---|
| 429 | |
---|
4 | 430 | for step in recipe: |
---|
2 | 431 | try: |
---|
2 | 432 | log.info('Executing build step %r, onerror = %s', step.id, step.onerror) |
---|
2 | 433 | if not self._execute_step(build_url, recipe, step): |
---|
0 | 434 | log.warning('Stopping build due to failure') |
---|
0 | 435 | break |
---|
0 | 436 | except Exception, e: |
---|
0 | 437 | log.error('Exception raised processing step %s. Reraising %s', step.id, e) |
---|
0 | 438 | raise |
---|
0 | 439 | else: |
---|
2 | 440 | log.info('Build completed') |
---|
2 | 441 | if self.dry_run: |
---|
0 | 442 | self._cancel_build(build_url) |
---|
0 | 443 | finally: |
---|
2 | 444 | if not self.local: |
---|
0 | 445 | keepalive_thread.stop() |
---|
2 | 446 | if not self.keep_files and os.path.isdir(basedir): |
---|
2 | 447 | log.debug('Removing build directory %s' % basedir) |
---|
2 | 448 | _rmtree(basedir) |
---|
2 | 449 | if self.single_build: |
---|
0 | 450 | log.info('Exiting after single build completed.') |
---|
0 | 451 | raise ExitSlave(EX_OK) |
---|
| 452 | |
---|
1 | 453 | def _execute_step(self, build_url, recipe, step): |
---|
2 | 454 | failed = False |
---|
2 | 455 | started = int(time.time()) |
---|
2 | 456 | xml = xmlio.Element('result', step=step.id) |
---|
2 | 457 | try: |
---|
2 | 458 | for type, category, generator, output in \ |
---|
4 | 459 | step.execute(recipe.ctxt): |
---|
2 | 460 | if type == Recipe.ERROR: |
---|
0 | 461 | failed = True |
---|
2 | 462 | if type == Recipe.REPORT and self.dump_reports: |
---|
0 | 463 | print output |
---|
2 | 464 | if type == Recipe.ATTACH: |
---|
| 465 | # Attachments are added out-of-band due to major |
---|
| 466 | # performance issues with inlined base64 xml content |
---|
0 | 467 | self._attach_file(build_url, recipe, output) |
---|
2 | 468 | xml.append(xmlio.Element(type, category=category, |
---|
2 | 469 | generator=generator)[ |
---|
2 | 470 | output |
---|
2 | 471 | ]) |
---|
0 | 472 | except KeyboardInterrupt: |
---|
0 | 473 | log.warning('Build interrupted') |
---|
0 | 474 | self._cancel_build(build_url) |
---|
0 | 475 | except BuildError, e: |
---|
0 | 476 | log.error('Build step %r failed', step.id) |
---|
0 | 477 | failed = True |
---|
0 | 478 | except Exception, e: |
---|
0 | 479 | log.error('Internal error in build step %r', step.id, exc_info=True) |
---|
0 | 480 | failed = True |
---|
2 | 481 | xml.attr['duration'] = (time.time() - started) |
---|
2 | 482 | if failed: |
---|
0 | 483 | xml.attr['status'] = 'failure' |
---|
0 | 484 | else: |
---|
2 | 485 | xml.attr['status'] = 'success' |
---|
2 | 486 | log.info('Build step %s completed successfully', step.id) |
---|
| 487 | |
---|
2 | 488 | if not self.local and not self.dry_run: |
---|
2 | 489 | try: |
---|
2 | 490 | resp = self.request('POST', build_url + '/steps/', str(xml), { |
---|
2 | 491 | 'Content-Type': 'application/x-bitten+xml' |
---|
2 | 492 | }) |
---|
2 | 493 | if resp.code != 201: |
---|
0 | 494 | log.error('Unexpected response (%d): %s', resp.code, |
---|
0 | 495 | resp.msg) |
---|
0 | 496 | except KeyboardInterrupt: |
---|
0 | 497 | log.warning('Build interrupted') |
---|
0 | 498 | self._cancel_build(build_url) |
---|
2 | 499 | return not failed or step.onerror != 'fail' |
---|
| 500 | |
---|
1 | 501 | def _cancel_build(self, build_url, exit_code=EX_OK): |
---|
0 | 502 | log.info('Cancelling build at %s', build_url) |
---|
0 | 503 | if not self.local: |
---|
0 | 504 | resp = self.request('DELETE', build_url) |
---|
0 | 505 | if resp.code not in (200, 204): |
---|
0 | 506 | log.error('Unexpected response (%d): %s', resp.code, resp.msg) |
---|
0 | 507 | raise ExitSlave(exit_code) |
---|
| 508 | |
---|
1 | 509 | def _attach_file(self, build_url, recipe, attachment): |
---|
0 | 510 | form_token = recipe._root.attr.get('form_token', '') |
---|
0 | 511 | if self.local or self.dry_run or not form_token: |
---|
0 | 512 | log.info('Attachment %s not sent due to current slave options', |
---|
0 | 513 | attachment.attr['filename']) |
---|
0 | 514 | return |
---|
0 | 515 | resource_type = attachment.attr['resource'] |
---|
0 | 516 | url = str(build_url + '/attach/' + resource_type) |
---|
0 | 517 | path = recipe.ctxt.resolve(attachment.attr['filename']) |
---|
0 | 518 | filename = os.path.basename(path) |
---|
0 | 519 | log.debug('Attaching file %s to %s...', attachment.attr['filename'], |
---|
0 | 520 | resource_type) |
---|
0 | 521 | f = open(path, 'rb') |
---|
0 | 522 | try: |
---|
0 | 523 | data, content_type = encode_multipart_formdata({ |
---|
0 | 524 | 'file': (filename, f.read()), |
---|
0 | 525 | 'description': attachment.attr['description'], |
---|
0 | 526 | '__FORM_TOKEN': form_token}) |
---|
0 | 527 | finally: |
---|
0 | 528 | f.close() |
---|
0 | 529 | resp = self.request('POST', url , data, { |
---|
0 | 530 | 'Content-Type': content_type}) |
---|
0 | 531 | if not resp.code == 201: |
---|
0 | 532 | msg = 'Error attaching %s to %s' |
---|
0 | 533 | log.error(msg, filename, resource_type) |
---|
0 | 534 | raise BuildError(msg, filename, resource_type) |
---|
| 535 | |
---|
2 | 536 | class ExitSlave(Exception): |
---|
| 537 | """Exception used internally by the slave to signal that the slave process |
---|
| 538 | should be stopped. |
---|
1 | 539 | """ |
---|
1 | 540 | def __init__(self, exit_code): |
---|
1 | 541 | self.exit_code = exit_code |
---|
1 | 542 | Exception.__init__(self) |
---|
| 543 | |
---|
| 544 | |
---|
1 | 545 | def main(): |
---|
| 546 | """Main entry point for running the build slave.""" |
---|
0 | 547 | from bitten import __version__ as VERSION |
---|
0 | 548 | from optparse import OptionParser |
---|
| 549 | |
---|
0 | 550 | parser = OptionParser(usage='usage: %prog [options] url1 [url2] ...', |
---|
0 | 551 | version='%%prog %s' % VERSION) |
---|
0 | 552 | parser.add_option('--name', action='store', dest='name', |
---|
0 | 553 | help='name of this slave (defaults to host name)') |
---|
0 | 554 | parser.add_option('-f', '--config', action='store', dest='config', |
---|
0 | 555 | metavar='FILE', help='path to configuration file') |
---|
0 | 556 | parser.add_option('-u', '--user', dest='username', |
---|
0 | 557 | help='the username to use for authentication') |
---|
0 | 558 | parser.add_option('-p', '--password', dest='password', |
---|
0 | 559 | help='the password to use when authenticating') |
---|
0 | 560 | def _ask_password(option, opt_str, value, parser): |
---|
0 | 561 | from getpass import getpass |
---|
0 | 562 | parser.values.password = getpass('Password: ') |
---|
0 | 563 | parser.add_option('-P', '--ask-password', action='callback', |
---|
0 | 564 | callback=_ask_password, help='Prompt for password') |
---|
0 | 565 | parser.add_option('--form-auth', action='store_true', |
---|
0 | 566 | dest='form_auth', |
---|
0 | 567 | help='login using AccountManager HTML form instead of ' |
---|
0 | 568 | 'HTTP authentication for all urls') |
---|
| 569 | |
---|
0 | 570 | group = parser.add_option_group('building') |
---|
0 | 571 | group.add_option('-d', '--work-dir', action='store', dest='work_dir', |
---|
0 | 572 | metavar='DIR', help='working directory for builds') |
---|
0 | 573 | group.add_option('--build-dir', action='store', dest='build_dir', |
---|
0 | 574 | default = 'build_${config}_${build}', |
---|
0 | 575 | help='name pattern for the build dir to use inside the ' |
---|
0 | 576 | 'working dir ["%default"]') |
---|
0 | 577 | group.add_option('-k', '--keep-files', action='store_true', |
---|
0 | 578 | dest='keep_files', |
---|
0 | 579 | help='don\'t delete files after builds') |
---|
0 | 580 | group.add_option('-s', '--single', action='store_true', |
---|
0 | 581 | dest='single_build', |
---|
0 | 582 | help='exit after completing a single build') |
---|
0 | 583 | group.add_option('', '--no-loop', action='store_true', |
---|
0 | 584 | dest='no_loop', |
---|
0 | 585 | help='exit after completing a single check and running ' |
---|
0 | 586 | 'the required builds') |
---|
0 | 587 | group.add_option('-n', '--dry-run', action='store_true', dest='dry_run', |
---|
0 | 588 | help='don\'t report results back to master') |
---|
0 | 589 | group.add_option('-i', '--interval', dest='interval', metavar='SECONDS', |
---|
0 | 590 | type='int', help='time to wait between requesting builds') |
---|
0 | 591 | group.add_option('-b', '--keepalive_interval', dest='keepalive_interval', metavar='SECONDS', type='int', help='time to wait between keepalive heartbeats') |
---|
0 | 592 | group = parser.add_option_group('logging') |
---|
0 | 593 | group.add_option('-l', '--log', dest='logfile', metavar='FILENAME', |
---|
0 | 594 | help='write log messages to FILENAME') |
---|
0 | 595 | group.add_option('-v', '--verbose', action='store_const', dest='loglevel', |
---|
0 | 596 | const=logging.DEBUG, help='print as much as possible') |
---|
0 | 597 | group.add_option('-q', '--quiet', action='store_const', dest='loglevel', |
---|
0 | 598 | const=logging.WARN, help='print as little as possible') |
---|
0 | 599 | group.add_option('--dump-reports', action='store_true', dest='dump_reports', |
---|
0 | 600 | help='whether report data should be printed') |
---|
| 601 | |
---|
0 | 602 | parser.set_defaults(dry_run=False, keep_files=False, |
---|
0 | 603 | loglevel=logging.INFO, single_build=False, no_loop=False, |
---|
0 | 604 | dump_reports=False, interval=300, keepalive_interval=60, |
---|
0 | 605 | form_auth=False) |
---|
0 | 606 | options, args = parser.parse_args() |
---|
| 607 | |
---|
0 | 608 | if len(args) < 1: |
---|
0 | 609 | parser.error('incorrect number of arguments') |
---|
0 | 610 | urls = args |
---|
| 611 | |
---|
0 | 612 | logger = logging.getLogger('bitten') |
---|
0 | 613 | logger.setLevel(options.loglevel) |
---|
0 | 614 | handler = logging.StreamHandler() |
---|
0 | 615 | handler.setLevel(options.loglevel) |
---|
0 | 616 | formatter = logging.Formatter('[%(levelname)-8s] %(message)s') |
---|
0 | 617 | handler.setFormatter(formatter) |
---|
0 | 618 | logger.addHandler(handler) |
---|
0 | 619 | if options.logfile: |
---|
0 | 620 | handler = logging.FileHandler(options.logfile) |
---|
0 | 621 | handler.setLevel(options.loglevel) |
---|
0 | 622 | formatter = logging.Formatter('%(asctime)s [%(name)s] %(levelname)s: ' |
---|
0 | 623 | '%(message)s') |
---|
0 | 624 | handler.setFormatter(formatter) |
---|
0 | 625 | logger.addHandler(handler) |
---|
| 626 | |
---|
0 | 627 | log.info("Slave launched at %s" % \ |
---|
0 | 628 | datetime.now().strftime('%Y-%m-%d %H:%M:%S')) |
---|
| 629 | |
---|
0 | 630 | slave = None |
---|
0 | 631 | try: |
---|
0 | 632 | slave = BuildSlave(urls, name=options.name, config=options.config, |
---|
0 | 633 | dry_run=options.dry_run, work_dir=options.work_dir, |
---|
0 | 634 | build_dir=options.build_dir, |
---|
0 | 635 | keep_files=options.keep_files, |
---|
0 | 636 | single_build=options.single_build, |
---|
0 | 637 | no_loop=options.no_loop, |
---|
0 | 638 | poll_interval=options.interval, |
---|
0 | 639 | keepalive_interval=options.keepalive_interval, |
---|
0 | 640 | username=options.username, password=options.password, |
---|
0 | 641 | dump_reports=options.dump_reports, |
---|
0 | 642 | form_auth=options.form_auth) |
---|
0 | 643 | try: |
---|
0 | 644 | exit_code = slave.run() |
---|
0 | 645 | except KeyboardInterrupt: |
---|
0 | 646 | slave.quit() |
---|
0 | 647 | except ConfigFileNotFound, e: |
---|
0 | 648 | log.error(e) |
---|
0 | 649 | exit_code = EX_IOERR |
---|
0 | 650 | except MissingSectionHeaderError: |
---|
0 | 651 | log.error("Error parsing configuration file %r. Wrong format?" \ |
---|
0 | 652 | % options.config) |
---|
0 | 653 | exit_code = EX_IOERR |
---|
0 | 654 | except ExitSlave, e: |
---|
0 | 655 | exit_code = e.exit_code |
---|
| 656 | |
---|
0 | 657 | if slave and not (options.work_dir or options.keep_files): |
---|
0 | 658 | log.debug('Removing working directory %s' % slave.work_dir) |
---|
0 | 659 | _rmtree(slave.work_dir) |
---|
| 660 | |
---|
0 | 661 | log.info("Slave exited at %s" % \ |
---|
0 | 662 | datetime.now().strftime('%Y-%m-%d %H:%M:%S')) |
---|
| 663 | |
---|
0 | 664 | return exit_code |
---|
| 665 | |
---|
1 | 666 | if __name__ == '__main__': |
---|
0 | 667 | sys.exit(main()) |
---|