1 | # -*- coding: utf-8 -*- |
---|
2 | # |
---|
3 | # Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de> |
---|
4 | # Copyright (C) 2008 Matt Good <matt@matt-good.net> |
---|
5 | # Copyright (C) 2008-2010 Edgewall Software |
---|
6 | # All rights reserved. |
---|
7 | # |
---|
8 | # This software is licensed as described in the file COPYING, which |
---|
9 | # you should have received as part of this distribution. The terms |
---|
10 | # are also available at http://bitten.edgewall.org/wiki/License. |
---|
11 | |
---|
12 | """Recipe commands for tools commonly used by Python projects.""" |
---|
13 | |
---|
14 | from __future__ import division |
---|
15 | |
---|
16 | import logging |
---|
17 | import os |
---|
18 | import cPickle as pickle |
---|
19 | import re |
---|
20 | try: |
---|
21 | set |
---|
22 | except NameError: |
---|
23 | from sets import Set as set |
---|
24 | import shlex |
---|
25 | import sys |
---|
26 | |
---|
27 | from bitten.build import CommandLine, FileSet |
---|
28 | from bitten.util import loc, xmlio |
---|
29 | |
---|
30 | log = logging.getLogger('bitten.build.pythontools') |
---|
31 | |
---|
32 | __docformat__ = 'restructuredtext en' |
---|
33 | |
---|
34 | def _python_path(ctxt): |
---|
35 | """Return the path to the Python interpreter. |
---|
36 | |
---|
37 | If the configuration has a ``python.path`` property, the value of that |
---|
38 | option is returned; otherwise the path to the current Python interpreter is |
---|
39 | returned. |
---|
40 | """ |
---|
41 | python_path = ctxt.config.get_filepath('python.path') |
---|
42 | if python_path: |
---|
43 | return python_path |
---|
44 | return sys.executable |
---|
45 | |
---|
46 | def distutils(ctxt, file_='setup.py', command='build', |
---|
47 | options=None, timeout=None): |
---|
48 | """Execute a ``distutils`` command. |
---|
49 | |
---|
50 | :param ctxt: the build context |
---|
51 | :type ctxt: `Context` |
---|
52 | :param file\_: name of the file defining the distutils setup |
---|
53 | :param command: the setup command to execute |
---|
54 | :param options: additional options to pass to the command |
---|
55 | :param timeout: the number of seconds before the external process should |
---|
56 | be aborted (has same constraints as CommandLine) |
---|
57 | """ |
---|
58 | if options: |
---|
59 | if isinstance(options, basestring): |
---|
60 | options = shlex.split(options) |
---|
61 | else: |
---|
62 | options = [] |
---|
63 | |
---|
64 | if timeout: |
---|
65 | timeout = int(timeout) |
---|
66 | |
---|
67 | cmdline = CommandLine(_python_path(ctxt), |
---|
68 | [ctxt.resolve(file_), command] + options, |
---|
69 | cwd=ctxt.basedir) |
---|
70 | log_elem = xmlio.Fragment() |
---|
71 | error_logged = False |
---|
72 | for out, err in cmdline.execute(timeout): |
---|
73 | if out is not None: |
---|
74 | log.info(out) |
---|
75 | log_elem.append(xmlio.Element('message', level='info')[out]) |
---|
76 | if err is not None: |
---|
77 | level = 'error' |
---|
78 | if err.startswith('warning: '): |
---|
79 | err = err[9:] |
---|
80 | level = 'warning' |
---|
81 | log.warning(err) |
---|
82 | elif err.startswith('error: '): |
---|
83 | ctxt.error(err[7:]) |
---|
84 | error_logged = True |
---|
85 | else: |
---|
86 | log.error(err) |
---|
87 | log_elem.append(xmlio.Element('message', level=level)[err]) |
---|
88 | ctxt.log(log_elem) |
---|
89 | |
---|
90 | if not error_logged and cmdline.returncode != 0: |
---|
91 | ctxt.error('distutils failed (%s)' % cmdline.returncode) |
---|
92 | |
---|
93 | def exec_(ctxt, file_=None, module=None, function=None, |
---|
94 | output=None, args=None, timeout=None): |
---|
95 | """Execute a Python script. |
---|
96 | |
---|
97 | Either the `file_` or the `module` parameter must be provided. If |
---|
98 | specified using the `file_` parameter, the file must be inside the project |
---|
99 | directory. If specified as a module, the module must either be resolvable |
---|
100 | to a file, or the `function` parameter must be provided |
---|
101 | |
---|
102 | :param ctxt: the build context |
---|
103 | :type ctxt: `Context` |
---|
104 | :param file\_: name of the script file to execute |
---|
105 | :param module: name of the Python module to execute |
---|
106 | :param function: name of the Python function to run |
---|
107 | :param output: name of the file to which output should be written |
---|
108 | :param args: extra arguments to pass to the script |
---|
109 | :param timeout: the number of seconds before the external process should |
---|
110 | be aborted (has same constraints as CommandLine) |
---|
111 | """ |
---|
112 | assert file_ or module, 'Either "file" or "module" attribute required' |
---|
113 | if function: |
---|
114 | assert module and not file_, '"module" attribute required for use of ' \ |
---|
115 | '"function" attribute' |
---|
116 | |
---|
117 | if module: |
---|
118 | # Script specified as module name, need to resolve that to a file, |
---|
119 | # or use the function name if provided |
---|
120 | if function: |
---|
121 | args = '-c "import sys; from %s import %s; %s(sys.argv)" %s' % ( |
---|
122 | module, function, function, args) |
---|
123 | else: |
---|
124 | try: |
---|
125 | mod = __import__(module, globals(), locals(), []) |
---|
126 | components = module.split('.') |
---|
127 | for comp in components[1:]: |
---|
128 | mod = getattr(mod, comp) |
---|
129 | file_ = mod.__file__.replace('\\', '/') |
---|
130 | except ImportError, e: |
---|
131 | ctxt.error('Cannot execute Python module %s: %s' % (module, e)) |
---|
132 | return |
---|
133 | |
---|
134 | from bitten.build import shtools |
---|
135 | returncode = shtools.execute(ctxt, executable=_python_path(ctxt), |
---|
136 | file_=file_, output=output, args=args, |
---|
137 | timeout=timeout) |
---|
138 | if returncode != 0: |
---|
139 | ctxt.error('Executing %s failed (error code %s)' % \ |
---|
140 | (file_ or function or module, returncode)) |
---|
141 | |
---|
142 | def pylint(ctxt, file_=None): |
---|
143 | """Extract data from a ``pylint`` run written to a file. |
---|
144 | |
---|
145 | :param ctxt: the build context |
---|
146 | :type ctxt: `Context` |
---|
147 | :param file\_: name of the file containing the Pylint output |
---|
148 | """ |
---|
149 | assert file_, 'Missing required attribute "file"' |
---|
150 | msg_re = re.compile(r'^(?P<file>.+):(?P<line>\d+): ' |
---|
151 | r'\[(?P<type>[A-Z]\d*)(?:, (?P<tag>[\w\.]+))?\] ' |
---|
152 | r'(?P<msg>.*)$') |
---|
153 | msg_categories = dict(W='warning', E='error', C='convention', R='refactor') |
---|
154 | |
---|
155 | problems = xmlio.Fragment() |
---|
156 | try: |
---|
157 | fd = open(ctxt.resolve(file_), 'r') |
---|
158 | try: |
---|
159 | for line in fd: |
---|
160 | match = msg_re.search(line) |
---|
161 | if match: |
---|
162 | msg_type = match.group('type') |
---|
163 | category = msg_categories.get(msg_type[0]) |
---|
164 | if len(msg_type) == 1: |
---|
165 | msg_type = None |
---|
166 | filename = match.group('file') |
---|
167 | if os.path.isabs(filename) \ |
---|
168 | and filename.startswith(ctxt.basedir): |
---|
169 | filename = filename[len(ctxt.basedir) + 1:] |
---|
170 | filename = filename.replace('\\', '/') |
---|
171 | lineno = int(match.group('line')) |
---|
172 | tag = match.group('tag') |
---|
173 | problems.append(xmlio.Element('problem', category=category, |
---|
174 | type=msg_type, tag=tag, |
---|
175 | line=lineno, file=filename)[ |
---|
176 | xmlio.Element('msg')[match.group('msg') or ''] |
---|
177 | ]) |
---|
178 | ctxt.report('lint', problems) |
---|
179 | finally: |
---|
180 | fd.close() |
---|
181 | except IOError, e: |
---|
182 | log.warning('Error opening pylint results file (%s)', e) |
---|
183 | |
---|
184 | def coverage(ctxt, summary=None, coverdir=None, include=None, exclude=None): |
---|
185 | """Extract data from a ``coverage.py`` run. |
---|
186 | |
---|
187 | :param ctxt: the build context |
---|
188 | :type ctxt: `Context` |
---|
189 | :param summary: path to the file containing the coverage summary |
---|
190 | :param coverdir: name of the directory containing the per-module coverage |
---|
191 | details |
---|
192 | :param include: patterns of files or directories to include in the report |
---|
193 | :param exclude: patterns of files or directories to exclude from the report |
---|
194 | """ |
---|
195 | assert summary, 'Missing required attribute "summary"' |
---|
196 | |
---|
197 | summary_line_re = re.compile(r'^(?P<module>.*?)\s+(?P<stmts>\d+)\s+' |
---|
198 | r'(?P<exec>\d+)\s+(?P<cov>\d+)%\s+' |
---|
199 | r'(?:(?P<missing>(?:\d+(?:-\d+)?(?:, )?)*)\s+)?' |
---|
200 | r'(?P<file>.+)$') |
---|
201 | |
---|
202 | fileset = FileSet(ctxt.basedir, include, exclude) |
---|
203 | missing_files = [] |
---|
204 | for filename in fileset: |
---|
205 | if os.path.splitext(filename)[1] != '.py': |
---|
206 | continue |
---|
207 | missing_files.append(filename) |
---|
208 | covered_modules = set() |
---|
209 | |
---|
210 | def handle_file(element, modname): |
---|
211 | if not coverdir: |
---|
212 | return |
---|
213 | fp = ctxt.resolve(os.path.join(coverdir, modname.replace(".", "_") + |
---|
214 | ".py,cover")) |
---|
215 | if not os.path.exists(fp): |
---|
216 | log.info("No line by line coverage available for %s", modname) |
---|
217 | return |
---|
218 | try: |
---|
219 | with open(fp) as f: |
---|
220 | lines = [] |
---|
221 | for line in f: |
---|
222 | if line.startswith(">"): |
---|
223 | lines.append("1") |
---|
224 | elif line.startswith("!"): |
---|
225 | lines.append("0") |
---|
226 | else: |
---|
227 | lines.append("-") |
---|
228 | element.append(xmlio.Element('line_hits')[' '.join(lines)]) |
---|
229 | except Exception, e: |
---|
230 | log.info("Error while processing line by line coverage: %s", e) |
---|
231 | |
---|
232 | try: |
---|
233 | summary_file = open(ctxt.resolve(summary), 'r') |
---|
234 | try: |
---|
235 | coverage = xmlio.Fragment() |
---|
236 | for summary_line in summary_file: |
---|
237 | match = summary_line_re.search(summary_line) |
---|
238 | if match: |
---|
239 | modname = match.group(1) |
---|
240 | filename = match.group(6) |
---|
241 | if not os.path.isabs(filename): |
---|
242 | filename = os.path.normpath(os.path.join(ctxt.basedir, |
---|
243 | filename)) |
---|
244 | else: |
---|
245 | filename = os.path.realpath(filename) |
---|
246 | if not filename.startswith(ctxt.basedir): |
---|
247 | continue |
---|
248 | filename = filename[len(ctxt.basedir) + 1:] |
---|
249 | if not filename in fileset: |
---|
250 | continue |
---|
251 | |
---|
252 | percentage = int(match.group(4).rstrip('%')) |
---|
253 | num_lines = int(match.group(2)) |
---|
254 | |
---|
255 | missing_files.remove(filename) |
---|
256 | covered_modules.add(modname) |
---|
257 | module = xmlio.Element('coverage', name=modname, |
---|
258 | file=filename.replace(os.sep, '/'), |
---|
259 | percentage=percentage, |
---|
260 | lines=num_lines) |
---|
261 | coverage.append(module) |
---|
262 | handle_file(module, modname) |
---|
263 | |
---|
264 | for filename in missing_files: |
---|
265 | modname = os.path.splitext(filename.replace(os.sep, '.'))[0] |
---|
266 | if modname in covered_modules: |
---|
267 | continue |
---|
268 | covered_modules.add(modname) |
---|
269 | module = xmlio.Element('coverage', name=modname, |
---|
270 | file=filename.replace(os.sep, '/'), |
---|
271 | percentage=0) |
---|
272 | coverage.append(module) |
---|
273 | |
---|
274 | ctxt.report('coverage', coverage) |
---|
275 | finally: |
---|
276 | summary_file.close() |
---|
277 | except IOError, e: |
---|
278 | log.warning('Error opening coverage summary file (%s)', e) |
---|
279 | |
---|
280 | def trace(ctxt, summary=None, coverdir=None, include=None, exclude=None): |
---|
281 | """Extract data from a ``trace.py`` run. |
---|
282 | |
---|
283 | :param ctxt: the build context |
---|
284 | :type ctxt: `Context` |
---|
285 | :param summary: path to the file containing the coverage summary |
---|
286 | :param coverdir: name of the directory containing the per-module coverage |
---|
287 | details |
---|
288 | :param include: patterns of files or directories to include in the report |
---|
289 | :param exclude: patterns of files or directories to exclude from the report |
---|
290 | """ |
---|
291 | assert summary, 'Missing required attribute "summary"' |
---|
292 | assert coverdir, 'Missing required attribute "coverdir"' |
---|
293 | |
---|
294 | summary_line_re = re.compile(r'^\s*(?P<lines>\d+)\s+(?P<cov>\d+)%\s+' |
---|
295 | r'(?P<module>.*?)\s+\((?P<filename>.*?)\)') |
---|
296 | coverage_line_re = re.compile(r'\s*(?:(?P<hits>\d+): )?(?P<line>.*)') |
---|
297 | |
---|
298 | fileset = FileSet(ctxt.basedir, include, exclude) |
---|
299 | missing_files = [] |
---|
300 | for filename in fileset: |
---|
301 | if os.path.splitext(filename)[1] != '.py': |
---|
302 | continue |
---|
303 | missing_files.append(filename) |
---|
304 | covered_modules = set() |
---|
305 | |
---|
306 | def handle_file(elem, sourcefile, coverfile=None): |
---|
307 | code_lines = set() |
---|
308 | for lineno, linetype, line in loc.count(sourcefile): |
---|
309 | if linetype == loc.CODE: |
---|
310 | code_lines.add(lineno) |
---|
311 | num_covered = 0 |
---|
312 | lines = [] |
---|
313 | |
---|
314 | if coverfile: |
---|
315 | prev_hits = '0' |
---|
316 | for idx, coverline in enumerate(coverfile): |
---|
317 | match = coverage_line_re.search(coverline) |
---|
318 | if match: |
---|
319 | hits = match.group(1) |
---|
320 | if hits: # Line covered |
---|
321 | if hits != '0': |
---|
322 | num_covered += 1 |
---|
323 | lines.append(hits) |
---|
324 | prev_hits = hits |
---|
325 | elif coverline.startswith('>'): # Line not covered |
---|
326 | lines.append('0') |
---|
327 | prev_hits = '0' |
---|
328 | elif idx not in code_lines: # Not a code line |
---|
329 | lines.append('-') |
---|
330 | prev_hits = '0' |
---|
331 | else: # A code line not flagged by trace.py |
---|
332 | if prev_hits != '0': |
---|
333 | num_covered += 1 |
---|
334 | lines.append(prev_hits) |
---|
335 | |
---|
336 | elem.append(xmlio.Element('line_hits')[' '.join(lines)]) |
---|
337 | |
---|
338 | num_lines = not lines and len(code_lines) or \ |
---|
339 | len([l for l in lines if l != '-']) |
---|
340 | if num_lines: |
---|
341 | percentage = int(round(num_covered * 100 / num_lines)) |
---|
342 | else: |
---|
343 | percentage = 0 |
---|
344 | elem.attr['percentage'] = percentage |
---|
345 | elem.attr['lines'] = num_lines |
---|
346 | |
---|
347 | try: |
---|
348 | summary_file = open(ctxt.resolve(summary), 'r') |
---|
349 | try: |
---|
350 | coverage = xmlio.Fragment() |
---|
351 | for summary_line in summary_file: |
---|
352 | match = summary_line_re.search(summary_line) |
---|
353 | if match: |
---|
354 | modname = match.group(3) |
---|
355 | filename = match.group(4) |
---|
356 | if not os.path.isabs(filename): |
---|
357 | filename = os.path.normpath(os.path.join(ctxt.basedir, |
---|
358 | filename)) |
---|
359 | else: |
---|
360 | filename = os.path.realpath(filename) |
---|
361 | if not filename.startswith(ctxt.basedir): |
---|
362 | continue |
---|
363 | filename = filename[len(ctxt.basedir) + 1:] |
---|
364 | if not filename in fileset: |
---|
365 | continue |
---|
366 | |
---|
367 | missing_files.remove(filename) |
---|
368 | covered_modules.add(modname) |
---|
369 | module = xmlio.Element('coverage', name=modname, |
---|
370 | file=filename.replace(os.sep, '/')) |
---|
371 | sourcefile = file(ctxt.resolve(filename)) |
---|
372 | try: |
---|
373 | coverpath = ctxt.resolve(coverdir, modname + '.cover') |
---|
374 | if os.path.isfile(coverpath): |
---|
375 | coverfile = file(coverpath, 'r') |
---|
376 | else: |
---|
377 | log.warning('No coverage file for module %s at %s', |
---|
378 | modname, coverpath) |
---|
379 | coverfile = None |
---|
380 | try: |
---|
381 | handle_file(module, sourcefile, coverfile) |
---|
382 | finally: |
---|
383 | if coverfile: |
---|
384 | coverfile.close() |
---|
385 | finally: |
---|
386 | sourcefile.close() |
---|
387 | coverage.append(module) |
---|
388 | |
---|
389 | for filename in missing_files: |
---|
390 | modname = os.path.splitext(filename.replace(os.sep, '.'))[0] |
---|
391 | if modname in covered_modules: |
---|
392 | continue |
---|
393 | covered_modules.add(modname) |
---|
394 | module = xmlio.Element('coverage', name=modname, |
---|
395 | file=filename.replace(os.sep, '/'), |
---|
396 | percentage=0) |
---|
397 | filepath = ctxt.resolve(filename) |
---|
398 | fileobj = file(filepath, 'r') |
---|
399 | try: |
---|
400 | handle_file(module, fileobj) |
---|
401 | finally: |
---|
402 | fileobj.close() |
---|
403 | coverage.append(module) |
---|
404 | |
---|
405 | ctxt.report('coverage', coverage) |
---|
406 | finally: |
---|
407 | summary_file.close() |
---|
408 | except IOError, e: |
---|
409 | log.warning('Error opening coverage summary file (%s)', e) |
---|
410 | |
---|
411 | def figleaf(ctxt, summary=None, include=None, exclude=None): |
---|
412 | """Extract data from a ``Figleaf`` run. |
---|
413 | |
---|
414 | :param ctxt: the build context |
---|
415 | :type ctxt: `Context` |
---|
416 | :param summary: path to the file containing the coverage summary |
---|
417 | :param include: patterns of files or directories to include in the report |
---|
418 | :param exclude: patterns of files or directories to exclude from the report |
---|
419 | """ |
---|
420 | from figleaf import get_lines |
---|
421 | coverage = xmlio.Fragment() |
---|
422 | try: |
---|
423 | fileobj = open(ctxt.resolve(summary)) |
---|
424 | except IOError, e: |
---|
425 | log.warning('Error opening coverage summary file (%s)', e) |
---|
426 | return |
---|
427 | coverage_data = pickle.load(fileobj) |
---|
428 | fileset = FileSet(ctxt.basedir, include, exclude) |
---|
429 | for filename in fileset: |
---|
430 | base, ext = os.path.splitext(filename) |
---|
431 | if ext != '.py': |
---|
432 | continue |
---|
433 | modname = base.replace(os.path.sep, '.') |
---|
434 | realfilename = ctxt.resolve(filename) |
---|
435 | interesting_lines = get_lines(open(realfilename)) |
---|
436 | if not interesting_lines: |
---|
437 | continue |
---|
438 | covered_lines = coverage_data.get(realfilename, set()) |
---|
439 | percentage = int(round(len(covered_lines) * 100 / len(interesting_lines))) |
---|
440 | line_hits = [] |
---|
441 | for lineno in xrange(1, max(interesting_lines)+1): |
---|
442 | if lineno not in interesting_lines: |
---|
443 | line_hits.append('-') |
---|
444 | elif lineno in covered_lines: |
---|
445 | line_hits.append('1') |
---|
446 | else: |
---|
447 | line_hits.append('0') |
---|
448 | module = xmlio.Element('coverage', name=modname, |
---|
449 | file=filename.replace(os.sep, '/'), |
---|
450 | percentage=percentage, |
---|
451 | lines=len(interesting_lines), |
---|
452 | line_hits=' '.join(line_hits)) |
---|
453 | coverage.append(module) |
---|
454 | ctxt.report('coverage', coverage) |
---|
455 | |
---|
456 | def _normalize_filenames(ctxt, filenames, fileset): |
---|
457 | for filename in filenames: |
---|
458 | if not os.path.isabs(filename): |
---|
459 | filename = os.path.normpath(os.path.join(ctxt.basedir, |
---|
460 | filename)) |
---|
461 | else: |
---|
462 | filename = os.path.realpath(filename) |
---|
463 | if not filename.startswith(ctxt.basedir): |
---|
464 | continue |
---|
465 | filename = filename[len(ctxt.basedir) + 1:] |
---|
466 | if filename not in fileset: |
---|
467 | continue |
---|
468 | yield filename.replace(os.sep, '/') |
---|
469 | |
---|
470 | def unittest(ctxt, file_=None): |
---|
471 | """Extract data from a unittest results file in XML format. |
---|
472 | |
---|
473 | :param ctxt: the build context |
---|
474 | :type ctxt: `Context` |
---|
475 | :param file\_: name of the file containing the test results |
---|
476 | """ |
---|
477 | assert file_, 'Missing required attribute "file"' |
---|
478 | |
---|
479 | try: |
---|
480 | fileobj = file(ctxt.resolve(file_), 'r') |
---|
481 | try: |
---|
482 | total, failed = 0, 0 |
---|
483 | results = xmlio.Fragment() |
---|
484 | for child in xmlio.parse(fileobj).children(): |
---|
485 | test = xmlio.Element('test') |
---|
486 | for name, value in child.attr.items(): |
---|
487 | if name == 'file': |
---|
488 | value = os.path.realpath(value) |
---|
489 | if value.startswith(ctxt.basedir): |
---|
490 | value = value[len(ctxt.basedir) + 1:] |
---|
491 | value = value.replace(os.sep, '/') |
---|
492 | else: |
---|
493 | continue |
---|
494 | test.attr[name] = value |
---|
495 | if name == 'status' and value in ('error', 'failure'): |
---|
496 | failed += 1 |
---|
497 | for grandchild in child.children(): |
---|
498 | test.append(xmlio.Element(grandchild.name)[ |
---|
499 | grandchild.gettext() |
---|
500 | ]) |
---|
501 | results.append(test) |
---|
502 | total += 1 |
---|
503 | if failed: |
---|
504 | ctxt.error('%d of %d test%s failed' % (failed, total, |
---|
505 | total != 1 and 's' or '')) |
---|
506 | ctxt.report('test', results) |
---|
507 | finally: |
---|
508 | fileobj.close() |
---|
509 | except IOError, e: |
---|
510 | log.warning('Error opening unittest results file (%s)', e) |
---|
511 | except xmlio.ParseError, e: |
---|
512 | log.warning('Error parsing unittest results file (%s)', e) |
---|