| 1 | # -*- coding: utf-8 -*- |
---|
| 2 | # |
---|
| 3 | # Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de> |
---|
| 4 | # Copyright (C) 2007-2010 Edgewall Software |
---|
| 5 | # All rights reserved. |
---|
| 6 | # |
---|
| 7 | # This software is licensed as described in the file COPYING, which |
---|
| 8 | # you should have received as part of this distribution. The terms |
---|
| 9 | # are also available at http://bitten.edgewall.org/wiki/License. |
---|
| 10 | |
---|
| 11 | """Functions and classes used to simplify the implementation recipe commands.""" |
---|
| 12 | |
---|
| 13 | import logging |
---|
| 14 | import fnmatch |
---|
| 15 | import os |
---|
| 16 | import shlex |
---|
| 17 | import time |
---|
| 18 | import subprocess |
---|
| 19 | import sys |
---|
| 20 | |
---|
| 21 | |
---|
| 22 | log = logging.getLogger('bitten.build.api') |
---|
| 23 | |
---|
| 24 | __docformat__ = 'restructuredtext en' |
---|
| 25 | |
---|
| 26 | |
---|
| 27 | class BuildError(Exception): |
---|
| 28 | """Exception raised when a build fails.""" |
---|
| 29 | |
---|
| 30 | |
---|
| 31 | class TimeoutError(Exception): |
---|
| 32 | """Exception raised when the execution of a command times out.""" |
---|
| 33 | |
---|
| 34 | |
---|
| 35 | def _encode(text): |
---|
| 36 | """Encode input for call. Input must be unicode or utf-8 string.""" |
---|
| 37 | if not isinstance(text, unicode): |
---|
| 38 | text = unicode(text, 'utf-8') |
---|
| 39 | # sys.stdin.encoding might be None (if stdin is directed from a file) |
---|
| 40 | # sys.stdin.encoding might be missing (if it is a StringIO object) |
---|
| 41 | encoding = sys.getfilesystemencoding() or \ |
---|
| 42 | getattr(sys.stdin, 'encoding', None) or 'utf-8' |
---|
| 43 | return text.encode(encoding, 'replace') |
---|
| 44 | |
---|
| 45 | def _decode(text): |
---|
| 46 | """Decode output from call.""" |
---|
| 47 | try: |
---|
| 48 | return text.decode('utf-8') |
---|
| 49 | except UnicodeDecodeError: |
---|
| 50 | # sys.stdout.encoding might be None (if stdout is directed to a file) |
---|
| 51 | # sys.stdout.encoding might be missing (if it is a StringIO object) |
---|
| 52 | encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' |
---|
| 53 | return text.decode(encoding, 'replace') |
---|
| 54 | |
---|
| 55 | |
---|
| 56 | class CommandLine(object): |
---|
| 57 | """Simple helper for executing subprocesses.""" |
---|
| 58 | |
---|
| 59 | def __init__(self, executable, args, input=None, cwd=None, shell=False): |
---|
| 60 | """Initialize the CommandLine object. |
---|
| 61 | |
---|
| 62 | :param executable: the name of the program to execute |
---|
| 63 | :param args: a list of arguments to pass to the executable |
---|
| 64 | :param input: string or file-like object containing any input data for |
---|
| 65 | the program |
---|
| 66 | :param cwd: the working directory to change to before executing the |
---|
| 67 | command |
---|
| 68 | """ |
---|
| 69 | self.executable = executable |
---|
| 70 | self.arguments = [_encode(arg) for arg in args] |
---|
| 71 | self.input = input |
---|
| 72 | self.cwd = cwd |
---|
| 73 | if self.cwd: |
---|
| 74 | assert os.path.isdir(self.cwd) |
---|
| 75 | self.shell = shell |
---|
| 76 | self.returncode = None |
---|
| 77 | |
---|
| 78 | |
---|
| 79 | def execute(self, timeout=None): |
---|
| 80 | """Execute the command, and return a generator for iterating over |
---|
| 81 | the output written to the standard output and error streams. |
---|
| 82 | |
---|
| 83 | :param timeout: number of seconds before the external process |
---|
| 84 | should be aborted (not supported on Windows without |
---|
| 85 | ``subprocess`` module / Python 2.4+) |
---|
| 86 | """ |
---|
| 87 | from threading import Thread |
---|
| 88 | from Queue import Queue, Empty |
---|
| 89 | |
---|
| 90 | def reader(pipe, pipe_name, queue): |
---|
| 91 | while pipe and not pipe.closed: |
---|
| 92 | line = pipe.readline() |
---|
| 93 | if line == '': |
---|
| 94 | break |
---|
| 95 | queue.put((pipe_name, line)) |
---|
| 96 | if not pipe.closed: |
---|
| 97 | pipe.close() |
---|
| 98 | |
---|
| 99 | def writer(pipe, data): |
---|
| 100 | if data and pipe and not pipe.closed: |
---|
| 101 | pipe.write(data) |
---|
| 102 | if not pipe.closed: |
---|
| 103 | pipe.close() |
---|
| 104 | |
---|
| 105 | args = [self.executable] + self.arguments |
---|
| 106 | try: |
---|
| 107 | p = subprocess.Popen(args, bufsize=1, # Line buffered |
---|
| 108 | stdin=subprocess.PIPE, |
---|
| 109 | stdout=subprocess.PIPE, |
---|
| 110 | stderr=subprocess.PIPE, |
---|
| 111 | cwd=(self.cwd or None), |
---|
| 112 | shell=self.shell, |
---|
| 113 | universal_newlines=True, |
---|
| 114 | env=None) |
---|
| 115 | except Exception, e: |
---|
| 116 | raise BuildError('Error executing %s: %s %s' % (args, |
---|
| 117 | e.__class__.__name__, str(e))) |
---|
| 118 | |
---|
| 119 | log.debug('Executing %s, (pid = %s, timeout = %s)', args, p.pid, timeout) |
---|
| 120 | |
---|
| 121 | if self.input: |
---|
| 122 | if isinstance(self.input, basestring): |
---|
| 123 | in_data = self.input |
---|
| 124 | else: |
---|
| 125 | in_data = self.input.read() |
---|
| 126 | else: |
---|
| 127 | in_data = None |
---|
| 128 | |
---|
| 129 | queue = Queue() |
---|
| 130 | limit = timeout and timeout + time.time() or 0 |
---|
| 131 | |
---|
| 132 | pipe_in = Thread(target=writer, args=(p.stdin, in_data)) |
---|
| 133 | pipe_out = Thread(target=reader, args=(p.stdout, 'stdout', queue)) |
---|
| 134 | pipe_err = Thread(target=reader, args=(p.stderr, 'stderr', queue)) |
---|
| 135 | pipe_err.start(); pipe_out.start(); pipe_in.start() |
---|
| 136 | |
---|
| 137 | while True: |
---|
| 138 | if limit and limit < time.time(): |
---|
| 139 | if hasattr(p, 'kill'): # Python 2.6+ |
---|
| 140 | log.debug('Killing process.') |
---|
| 141 | p.kill() |
---|
| 142 | raise TimeoutError('Command %s timed out' % self.executable) |
---|
| 143 | if p.poll() != None and self.returncode == None: |
---|
| 144 | self.returncode = p.returncode |
---|
| 145 | try: |
---|
| 146 | name, line = queue.get(block=True, timeout=.01) |
---|
| 147 | line = line and _decode(line.rstrip().replace('\x00', '')) |
---|
| 148 | if name == 'stderr': |
---|
| 149 | yield (None, line) |
---|
| 150 | else: |
---|
| 151 | yield (line, None) |
---|
| 152 | except Empty: |
---|
| 153 | if self.returncode != None: |
---|
| 154 | break |
---|
| 155 | |
---|
| 156 | pipe_out.join(); pipe_in.join(); pipe_err.join() |
---|
| 157 | |
---|
| 158 | log.debug('%s exited with code %s', self.executable, |
---|
| 159 | self.returncode) |
---|
| 160 | |
---|
| 161 | |
---|
| 162 | class FileSet(object): |
---|
| 163 | """Utility class for collecting a list of files in a directory that match |
---|
| 164 | given name/path patterns.""" |
---|
| 165 | |
---|
| 166 | DEFAULT_EXCLUDES = ['CVS/*', '*/CVS/*', '.svn/*', '*/.svn/*', |
---|
| 167 | '.DS_Store', 'Thumbs.db'] |
---|
| 168 | |
---|
| 169 | def __init__(self, basedir, include=None, exclude=None): |
---|
| 170 | """Create a file set. |
---|
| 171 | |
---|
| 172 | :param basedir: the base directory for all files in the set |
---|
| 173 | :param include: a list of patterns that define which files should be |
---|
| 174 | included in the set |
---|
| 175 | :param exclude: a list of patterns that define which files should be |
---|
| 176 | excluded from the set |
---|
| 177 | """ |
---|
| 178 | self.files = [] |
---|
| 179 | self.basedir = basedir |
---|
| 180 | |
---|
| 181 | self.include = [] |
---|
| 182 | if include is not None: |
---|
| 183 | self.include = shlex.split(include) |
---|
| 184 | |
---|
| 185 | self.exclude = self.DEFAULT_EXCLUDES[:] |
---|
| 186 | if exclude is not None: |
---|
| 187 | self.exclude += shlex.split(exclude) |
---|
| 188 | |
---|
| 189 | for dirpath, dirnames, filenames in os.walk(self.basedir): |
---|
| 190 | dirpath = dirpath[len(self.basedir) + 1:] |
---|
| 191 | |
---|
| 192 | for filename in filenames: |
---|
| 193 | filepath = nfilepath = os.path.join(dirpath, filename) |
---|
| 194 | if os.sep != '/': |
---|
| 195 | nfilepath = nfilepath.replace(os.sep, '/') |
---|
| 196 | |
---|
| 197 | if self.include: |
---|
| 198 | included = False |
---|
| 199 | for pattern in self.include: |
---|
| 200 | if fnmatch.fnmatchcase(nfilepath, pattern) or \ |
---|
| 201 | fnmatch.fnmatchcase(filename, pattern): |
---|
| 202 | included = True |
---|
| 203 | break |
---|
| 204 | if not included: |
---|
| 205 | continue |
---|
| 206 | |
---|
| 207 | excluded = False |
---|
| 208 | for pattern in self.exclude: |
---|
| 209 | if fnmatch.fnmatchcase(nfilepath, pattern) or \ |
---|
| 210 | fnmatch.fnmatchcase(filename, pattern): |
---|
| 211 | excluded = True |
---|
| 212 | break |
---|
| 213 | if not excluded: |
---|
| 214 | self.files.append(filepath) |
---|
| 215 | |
---|
| 216 | def __iter__(self): |
---|
| 217 | """Iterate over the names of all files in the set.""" |
---|
| 218 | for filename in self.files: |
---|
| 219 | yield filename |
---|
| 220 | |
---|
| 221 | def __contains__(self, filename): |
---|
| 222 | """Return whether the given file name is in the set. |
---|
| 223 | |
---|
| 224 | :param filename: the name of the file to check |
---|
| 225 | """ |
---|
| 226 | return filename in self.files |
---|