1 | # -*- coding: utf-8 -*- |
---|
2 | # |
---|
3 | # Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de> |
---|
4 | # Copyright (C) 2007-2010 Edgewall Software |
---|
5 | # All rights reserved. |
---|
6 | # |
---|
7 | # This software is licensed as described in the file COPYING, which |
---|
8 | # you should have received as part of this distribution. The terms |
---|
9 | # are also available at http://bitten.edgewall.org/wiki/License. |
---|
10 | |
---|
11 | """Functions and classes used to simplify the implementation recipe commands.""" |
---|
12 | |
---|
13 | import logging |
---|
14 | import fnmatch |
---|
15 | import os |
---|
16 | import shlex |
---|
17 | import time |
---|
18 | import subprocess |
---|
19 | import sys |
---|
20 | |
---|
21 | |
---|
22 | log = logging.getLogger('bitten.build.api') |
---|
23 | |
---|
24 | __docformat__ = 'restructuredtext en' |
---|
25 | |
---|
26 | |
---|
27 | class BuildError(Exception): |
---|
28 | """Exception raised when a build fails.""" |
---|
29 | |
---|
30 | |
---|
31 | class TimeoutError(Exception): |
---|
32 | """Exception raised when the execution of a command times out.""" |
---|
33 | |
---|
34 | |
---|
35 | def _encode(text): |
---|
36 | """Encode input for call. Input must be unicode or utf-8 string.""" |
---|
37 | if not isinstance(text, unicode): |
---|
38 | text = unicode(text, 'utf-8') |
---|
39 | # sys.stdin.encoding might be None (if stdin is directed from a file) |
---|
40 | # sys.stdin.encoding might be missing (if it is a StringIO object) |
---|
41 | encoding = sys.getfilesystemencoding() or \ |
---|
42 | getattr(sys.stdin, 'encoding', None) or 'utf-8' |
---|
43 | return text.encode(encoding, 'replace') |
---|
44 | |
---|
45 | def _decode(text): |
---|
46 | """Decode output from call.""" |
---|
47 | try: |
---|
48 | return text.decode('utf-8') |
---|
49 | except UnicodeDecodeError: |
---|
50 | # sys.stdout.encoding might be None (if stdout is directed to a file) |
---|
51 | # sys.stdout.encoding might be missing (if it is a StringIO object) |
---|
52 | encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' |
---|
53 | return text.decode(encoding, 'replace') |
---|
54 | |
---|
55 | |
---|
56 | class CommandLine(object): |
---|
57 | """Simple helper for executing subprocesses.""" |
---|
58 | |
---|
59 | def __init__(self, executable, args, input=None, cwd=None, shell=False): |
---|
60 | """Initialize the CommandLine object. |
---|
61 | |
---|
62 | :param executable: the name of the program to execute |
---|
63 | :param args: a list of arguments to pass to the executable |
---|
64 | :param input: string or file-like object containing any input data for |
---|
65 | the program |
---|
66 | :param cwd: the working directory to change to before executing the |
---|
67 | command |
---|
68 | """ |
---|
69 | self.executable = executable |
---|
70 | self.arguments = [_encode(arg) for arg in args] |
---|
71 | self.input = input |
---|
72 | self.cwd = cwd |
---|
73 | if self.cwd: |
---|
74 | assert os.path.isdir(self.cwd) |
---|
75 | self.shell = shell |
---|
76 | self.returncode = None |
---|
77 | |
---|
78 | |
---|
79 | def execute(self, timeout=None): |
---|
80 | """Execute the command, and return a generator for iterating over |
---|
81 | the output written to the standard output and error streams. |
---|
82 | |
---|
83 | :param timeout: number of seconds before the external process |
---|
84 | should be aborted (not supported on Windows without |
---|
85 | ``subprocess`` module / Python 2.4+) |
---|
86 | """ |
---|
87 | from threading import Thread |
---|
88 | from Queue import Queue, Empty |
---|
89 | |
---|
90 | def reader(pipe, pipe_name, queue): |
---|
91 | while pipe and not pipe.closed: |
---|
92 | line = pipe.readline() |
---|
93 | if line == '': |
---|
94 | break |
---|
95 | queue.put((pipe_name, line)) |
---|
96 | if not pipe.closed: |
---|
97 | pipe.close() |
---|
98 | |
---|
99 | def writer(pipe, data): |
---|
100 | if data and pipe and not pipe.closed: |
---|
101 | pipe.write(data) |
---|
102 | if not pipe.closed: |
---|
103 | pipe.close() |
---|
104 | |
---|
105 | args = [self.executable] + self.arguments |
---|
106 | try: |
---|
107 | p = subprocess.Popen(args, bufsize=1, # Line buffered |
---|
108 | stdin=subprocess.PIPE, |
---|
109 | stdout=subprocess.PIPE, |
---|
110 | stderr=subprocess.PIPE, |
---|
111 | cwd=(self.cwd or None), |
---|
112 | shell=self.shell, |
---|
113 | universal_newlines=True, |
---|
114 | env=None) |
---|
115 | except Exception, e: |
---|
116 | raise BuildError('Error executing %s: %s %s' % (args, |
---|
117 | e.__class__.__name__, str(e))) |
---|
118 | |
---|
119 | log.debug('Executing %s, (pid = %s, timeout = %s)', args, p.pid, timeout) |
---|
120 | |
---|
121 | if self.input: |
---|
122 | if isinstance(self.input, basestring): |
---|
123 | in_data = self.input |
---|
124 | else: |
---|
125 | in_data = self.input.read() |
---|
126 | else: |
---|
127 | in_data = None |
---|
128 | |
---|
129 | queue = Queue() |
---|
130 | limit = timeout and timeout + time.time() or 0 |
---|
131 | |
---|
132 | pipe_in = Thread(target=writer, args=(p.stdin, in_data)) |
---|
133 | pipe_out = Thread(target=reader, args=(p.stdout, 'stdout', queue)) |
---|
134 | pipe_err = Thread(target=reader, args=(p.stderr, 'stderr', queue)) |
---|
135 | pipe_err.start(); pipe_out.start(); pipe_in.start() |
---|
136 | |
---|
137 | while True: |
---|
138 | if limit and limit < time.time(): |
---|
139 | if hasattr(p, 'kill'): # Python 2.6+ |
---|
140 | log.debug('Killing process.') |
---|
141 | p.kill() |
---|
142 | raise TimeoutError('Command %s timed out' % self.executable) |
---|
143 | if p.poll() != None and self.returncode == None: |
---|
144 | self.returncode = p.returncode |
---|
145 | try: |
---|
146 | name, line = queue.get(block=True, timeout=.01) |
---|
147 | line = line and _decode(line.rstrip().replace('\x00', '')) |
---|
148 | if name == 'stderr': |
---|
149 | yield (None, line) |
---|
150 | else: |
---|
151 | yield (line, None) |
---|
152 | except Empty: |
---|
153 | if self.returncode != None: |
---|
154 | break |
---|
155 | |
---|
156 | pipe_out.join(); pipe_in.join(); pipe_err.join() |
---|
157 | |
---|
158 | log.debug('%s exited with code %s', self.executable, |
---|
159 | self.returncode) |
---|
160 | |
---|
161 | |
---|
162 | class FileSet(object): |
---|
163 | """Utility class for collecting a list of files in a directory that match |
---|
164 | given name/path patterns.""" |
---|
165 | |
---|
166 | DEFAULT_EXCLUDES = ['CVS/*', '*/CVS/*', '.svn/*', '*/.svn/*', |
---|
167 | '.DS_Store', 'Thumbs.db'] |
---|
168 | |
---|
169 | def __init__(self, basedir, include=None, exclude=None): |
---|
170 | """Create a file set. |
---|
171 | |
---|
172 | :param basedir: the base directory for all files in the set |
---|
173 | :param include: a list of patterns that define which files should be |
---|
174 | included in the set |
---|
175 | :param exclude: a list of patterns that define which files should be |
---|
176 | excluded from the set |
---|
177 | """ |
---|
178 | self.files = [] |
---|
179 | self.basedir = basedir |
---|
180 | |
---|
181 | self.include = [] |
---|
182 | if include is not None: |
---|
183 | self.include = shlex.split(include) |
---|
184 | |
---|
185 | self.exclude = self.DEFAULT_EXCLUDES[:] |
---|
186 | if exclude is not None: |
---|
187 | self.exclude += shlex.split(exclude) |
---|
188 | |
---|
189 | for dirpath, dirnames, filenames in os.walk(self.basedir): |
---|
190 | dirpath = dirpath[len(self.basedir) + 1:] |
---|
191 | |
---|
192 | for filename in filenames: |
---|
193 | filepath = nfilepath = os.path.join(dirpath, filename) |
---|
194 | if os.sep != '/': |
---|
195 | nfilepath = nfilepath.replace(os.sep, '/') |
---|
196 | |
---|
197 | if self.include: |
---|
198 | included = False |
---|
199 | for pattern in self.include: |
---|
200 | if fnmatch.fnmatchcase(nfilepath, pattern) or \ |
---|
201 | fnmatch.fnmatchcase(filename, pattern): |
---|
202 | included = True |
---|
203 | break |
---|
204 | if not included: |
---|
205 | continue |
---|
206 | |
---|
207 | excluded = False |
---|
208 | for pattern in self.exclude: |
---|
209 | if fnmatch.fnmatchcase(nfilepath, pattern) or \ |
---|
210 | fnmatch.fnmatchcase(filename, pattern): |
---|
211 | excluded = True |
---|
212 | break |
---|
213 | if not excluded: |
---|
214 | self.files.append(filepath) |
---|
215 | |
---|
216 | def __iter__(self): |
---|
217 | """Iterate over the names of all files in the set.""" |
---|
218 | for filename in self.files: |
---|
219 | yield filename |
---|
220 | |
---|
221 | def __contains__(self, filename): |
---|
222 | """Return whether the given file name is in the set. |
---|
223 | |
---|
224 | :param filename: the name of the file to check |
---|
225 | """ |
---|
226 | return filename in self.files |
---|