Edgewall Software
close Warning: No coverage annotation found for /trunk/bitten/build/api.py for revision range [1001:1001].

source: trunk/bitten/build/api.py @ 1001

Last change on this file since 1001 was 923, checked in by hodgestar, 13 years ago

Correctly handle the various possible values of sys.stdout/in.encoding.

  • Property svn:eol-style set to native
File size: 7.9 KB
CovLine 
1# -*- coding: utf-8 -*-
2#
3# Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de>
4# Copyright (C) 2007-2010 Edgewall Software
5# All rights reserved.
6#
7# This software is licensed as described in the file COPYING, which
8# you should have received as part of this distribution. The terms
9# are also available at http://bitten.edgewall.org/wiki/License.
10
11"""Functions and classes used to simplify the implementation recipe commands."""
12
13import logging
14import fnmatch
15import os
16import shlex
17import time
18import subprocess
19import sys
20
21
22log = logging.getLogger('bitten.build.api')
23
24__docformat__ = 'restructuredtext en'
25
26
27class BuildError(Exception):
28    """Exception raised when a build fails."""
29
30
31class TimeoutError(Exception):
32    """Exception raised when the execution of a command times out."""
33
34
35def _encode(text):
36    """Encode input for call. Input must be unicode or utf-8 string."""
37    if not isinstance(text, unicode):
38        text = unicode(text, 'utf-8')
39    # sys.stdin.encoding might be None (if stdin is directed from a file)
40    # sys.stdin.encoding might be missing (if it is a StringIO object)
41    encoding = sys.getfilesystemencoding() or \
42                getattr(sys.stdin, 'encoding', None) or 'utf-8'
43    return text.encode(encoding, 'replace')
44
45def _decode(text):
46    """Decode output from call."""
47    try:
48        return text.decode('utf-8')
49    except UnicodeDecodeError:
50        # sys.stdout.encoding might be None (if stdout is directed to a file)
51        # sys.stdout.encoding might be missing (if it is a StringIO object)
52        encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8'
53        return text.decode(encoding, 'replace')
54
55
56class CommandLine(object):
57    """Simple helper for executing subprocesses."""
58
59    def __init__(self, executable, args, input=None, cwd=None, shell=False):
60        """Initialize the CommandLine object.
61       
62        :param executable: the name of the program to execute
63        :param args: a list of arguments to pass to the executable
64        :param input: string or file-like object containing any input data for
65                      the program
66        :param cwd: the working directory to change to before executing the
67                    command
68        """
69        self.executable = executable
70        self.arguments = [_encode(arg) for arg in args]
71        self.input = input
72        self.cwd = cwd
73        if self.cwd:
74            assert os.path.isdir(self.cwd)
75        self.shell = shell
76        self.returncode = None
77
78
79    def execute(self, timeout=None):
80        """Execute the command, and return a generator for iterating over
81        the output written to the standard output and error streams.
82       
83        :param timeout: number of seconds before the external process
84                        should be aborted (not supported on Windows without
85                        ``subprocess`` module / Python 2.4+)
86        """
87        from threading import Thread
88        from Queue import Queue, Empty
89
90        def reader(pipe, pipe_name, queue):
91            while pipe and not pipe.closed:
92                line = pipe.readline()
93                if line == '':
94                    break
95                queue.put((pipe_name, line))
96            if not pipe.closed:
97                pipe.close()
98
99        def writer(pipe, data):
100            if data and pipe and not pipe.closed:
101                pipe.write(data)
102            if not pipe.closed:
103                pipe.close()
104
105        args = [self.executable] + self.arguments
106        try:
107            p = subprocess.Popen(args, bufsize=1, # Line buffered
108                        stdin=subprocess.PIPE,
109                        stdout=subprocess.PIPE,
110                        stderr=subprocess.PIPE,
111                        cwd=(self.cwd or None),
112                        shell=self.shell,
113                        universal_newlines=True,
114                        env=None)
115        except Exception, e:
116            raise BuildError('Error executing %s: %s %s' % (args,
117                                        e.__class__.__name__, str(e)))
118
119        log.debug('Executing %s, (pid = %s, timeout = %s)', args, p.pid, timeout)
120
121        if self.input:
122            if isinstance(self.input, basestring):
123                in_data = self.input
124            else:
125                in_data = self.input.read()
126        else:
127            in_data = None
128       
129        queue = Queue()
130        limit = timeout and timeout + time.time() or 0
131
132        pipe_in = Thread(target=writer, args=(p.stdin, in_data))
133        pipe_out = Thread(target=reader, args=(p.stdout, 'stdout', queue))
134        pipe_err = Thread(target=reader, args=(p.stderr, 'stderr', queue))
135        pipe_err.start(); pipe_out.start(); pipe_in.start()
136
137        while True:
138            if limit and limit < time.time():
139                if hasattr(p, 'kill'): # Python 2.6+
140                    log.debug('Killing process.')
141                    p.kill()
142                raise TimeoutError('Command %s timed out' % self.executable)
143            if p.poll() != None and self.returncode == None:
144                self.returncode = p.returncode
145            try:
146                name, line = queue.get(block=True, timeout=.01)
147                line = line and _decode(line.rstrip().replace('\x00', ''))
148                if name == 'stderr':
149                    yield (None, line)
150                else:
151                    yield (line, None)
152            except Empty:
153                if self.returncode != None:
154                    break
155
156        pipe_out.join(); pipe_in.join(); pipe_err.join()
157
158        log.debug('%s exited with code %s', self.executable,
159                  self.returncode)
160
161
162class FileSet(object):
163    """Utility class for collecting a list of files in a directory that match
164    given name/path patterns."""
165
166    DEFAULT_EXCLUDES = ['CVS/*', '*/CVS/*', '.svn/*', '*/.svn/*',
167                        '.DS_Store', 'Thumbs.db']
168
169    def __init__(self, basedir, include=None, exclude=None):
170        """Create a file set.
171       
172        :param basedir: the base directory for all files in the set
173        :param include: a list of patterns that define which files should be
174                        included in the set
175        :param exclude: a list of patterns that define which files should be
176                        excluded from the set
177        """
178        self.files = []
179        self.basedir = basedir
180
181        self.include = []
182        if include is not None:
183            self.include = shlex.split(include)
184
185        self.exclude = self.DEFAULT_EXCLUDES[:]
186        if exclude is not None:
187            self.exclude += shlex.split(exclude)
188
189        for dirpath, dirnames, filenames in os.walk(self.basedir):
190            dirpath = dirpath[len(self.basedir) + 1:]
191
192            for filename in filenames:
193                filepath = nfilepath = os.path.join(dirpath, filename)
194                if os.sep != '/':
195                    nfilepath = nfilepath.replace(os.sep, '/')
196
197                if self.include:
198                    included = False
199                    for pattern in self.include:
200                        if fnmatch.fnmatchcase(nfilepath, pattern) or \
201                           fnmatch.fnmatchcase(filename, pattern):
202                            included = True
203                            break
204                    if not included:
205                        continue
206
207                excluded = False
208                for pattern in self.exclude:
209                    if fnmatch.fnmatchcase(nfilepath, pattern) or \
210                       fnmatch.fnmatchcase(filename, pattern):
211                        excluded = True
212                        break
213                if not excluded:
214                    self.files.append(filepath)
215
216    def __iter__(self):
217        """Iterate over the names of all files in the set."""
218        for filename in self.files:
219            yield filename
220
221    def __contains__(self, filename):
222        """Return whether the given file name is in the set.
223       
224        :param filename: the name of the file to check
225        """
226        return filename in self.files
Note: See TracBrowser for help on using the repository browser.