1 | # -*- coding: utf-8 -*- |
---|
2 | # |
---|
3 | # Copyright (C) 2007-2010 Edgewall Software |
---|
4 | # Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de> |
---|
5 | # All rights reserved. |
---|
6 | # |
---|
7 | # This software is licensed as described in the file COPYING, which |
---|
8 | # you should have received as part of this distribution. The terms |
---|
9 | # are also available at http://bitten.edgewall.org/wiki/License. |
---|
10 | |
---|
11 | """Implements the scheduling of builds for a project. |
---|
12 | |
---|
13 | This module provides the functionality for scheduling builds for a specific |
---|
14 | Trac environment. It is used by both the build master and the web interface to |
---|
15 | get the list of required builds (revisions not built yet). |
---|
16 | |
---|
17 | Furthermore, the `BuildQueue` class is used by the build master to determine |
---|
18 | the next pending build, and to match build slaves against configured target |
---|
19 | platforms. |
---|
20 | """ |
---|
21 | |
---|
22 | from itertools import ifilter |
---|
23 | import re |
---|
24 | import time |
---|
25 | |
---|
26 | from trac.util.datefmt import to_timestamp |
---|
27 | from trac.util import pretty_timedelta, format_datetime |
---|
28 | from trac.attachment import Attachment |
---|
29 | |
---|
30 | |
---|
31 | from bitten.model import BuildConfig, TargetPlatform, Build, BuildStep |
---|
32 | from bitten.util.repository import get_repos |
---|
33 | |
---|
34 | __docformat__ = 'restructuredtext en' |
---|
35 | |
---|
36 | |
---|
37 | def collect_changes(config, authname=None, db=None): |
---|
38 | """Collect all changes for a build configuration that either have already |
---|
39 | been built, or still need to be built. |
---|
40 | |
---|
41 | This function is a generator that yields ``(platform, rev, build)`` tuples, |
---|
42 | where ``platform`` is a `TargetPlatform` object, ``rev`` is the identifier |
---|
43 | of the changeset, and ``build`` is a `Build` object or `None`. |
---|
44 | |
---|
45 | :param config: the build configuration |
---|
46 | :param authname: the logged in user |
---|
47 | :param db: a database connection (optional) |
---|
48 | """ |
---|
49 | env = config.env |
---|
50 | |
---|
51 | repos_name, repos, repos_path = get_repos(env, config.path, authname) |
---|
52 | |
---|
53 | if not db: |
---|
54 | db = env.get_db_cnx() |
---|
55 | try: |
---|
56 | node = repos.get_node(repos_path) |
---|
57 | except Exception, e: |
---|
58 | env.log.warn('Error accessing path %r for configuration %r', |
---|
59 | repos_path, config.name, exc_info=True) |
---|
60 | return |
---|
61 | |
---|
62 | for path, rev, chg in node.get_history(): |
---|
63 | |
---|
64 | # Don't follow moves/copies |
---|
65 | if path != repos.normalize_path(repos_path): |
---|
66 | break |
---|
67 | |
---|
68 | # Stay within the limits of the build config |
---|
69 | if config.min_rev and repos.rev_older_than(rev, config.min_rev): |
---|
70 | break |
---|
71 | if config.max_rev and repos.rev_older_than(config.max_rev, rev): |
---|
72 | continue |
---|
73 | |
---|
74 | # Make sure the repository directory isn't empty at this |
---|
75 | # revision |
---|
76 | old_node = repos.get_node(path, rev) |
---|
77 | is_empty = True |
---|
78 | for entry in old_node.get_entries(): |
---|
79 | is_empty = False |
---|
80 | break |
---|
81 | if is_empty: |
---|
82 | continue |
---|
83 | |
---|
84 | # For every target platform, check whether there's a build |
---|
85 | # of this revision |
---|
86 | for platform in TargetPlatform.select(env, config.name, db=db): |
---|
87 | builds = list(Build.select(env, config.name, rev, platform.id, |
---|
88 | db=db)) |
---|
89 | if builds: |
---|
90 | build = builds[0] |
---|
91 | else: |
---|
92 | build = None |
---|
93 | |
---|
94 | yield platform, rev, build |
---|
95 | |
---|
96 | |
---|
97 | class BuildQueue(object): |
---|
98 | """Enapsulates the build queue of an environment. |
---|
99 | |
---|
100 | A build queue manages the the registration of build slaves and detection of |
---|
101 | repository revisions that need to be built. |
---|
102 | """ |
---|
103 | |
---|
104 | def __init__(self, env, build_all=False, stabilize_wait=0, timeout=0): |
---|
105 | """Create the build queue. |
---|
106 | |
---|
107 | :param env: the Trac environment |
---|
108 | :param build_all: whether older revisions should be built |
---|
109 | :param stabilize_wait: The time in seconds to wait before considering |
---|
110 | the repository stable to create a build in the queue. |
---|
111 | :param timeout: the time in seconds after which an in-progress build |
---|
112 | should be considered orphaned, and reset to pending |
---|
113 | state |
---|
114 | """ |
---|
115 | self.env = env |
---|
116 | self.log = env.log |
---|
117 | self.build_all = build_all |
---|
118 | self.stabilize_wait = stabilize_wait |
---|
119 | self.timeout = timeout |
---|
120 | |
---|
121 | # Build scheduling |
---|
122 | |
---|
123 | def get_build_for_slave(self, name, properties): |
---|
124 | """Check whether one of the pending builds can be built by the build |
---|
125 | slave. |
---|
126 | |
---|
127 | :param name: the name of the slave |
---|
128 | :type name: `basestring` |
---|
129 | :param properties: the slave configuration |
---|
130 | :type properties: `dict` |
---|
131 | :return: the allocated build, or `None` if no build was found |
---|
132 | :rtype: `Build` |
---|
133 | """ |
---|
134 | self.log.debug('Checking for pending builds...') |
---|
135 | |
---|
136 | db = self.env.get_db_cnx() |
---|
137 | |
---|
138 | self.reset_orphaned_builds() |
---|
139 | |
---|
140 | # Iterate through pending builds by descending revision timestamp, to |
---|
141 | # avoid the first configuration/platform getting all the builds |
---|
142 | platforms = [p.id for p in self.match_slave(name, properties)] |
---|
143 | builds_to_delete = [] |
---|
144 | build_found = False |
---|
145 | for build in Build.select(self.env, status=Build.PENDING, db=db): |
---|
146 | config_path = BuildConfig.fetch(self.env, name=build.config).path |
---|
147 | _name, repos, _path = get_repos(self.env, config_path, None) |
---|
148 | if self.should_delete_build(build, repos): |
---|
149 | self.log.info('Scheduling build %d for deletion', build.id) |
---|
150 | builds_to_delete.append(build) |
---|
151 | elif build.platform in platforms: |
---|
152 | build_found = True |
---|
153 | break |
---|
154 | if not build_found: |
---|
155 | self.log.debug('No pending builds.') |
---|
156 | build = None |
---|
157 | |
---|
158 | # delete any obsolete builds |
---|
159 | for build_to_delete in builds_to_delete: |
---|
160 | build_to_delete.delete(db=db) |
---|
161 | |
---|
162 | if build: |
---|
163 | build.slave = name |
---|
164 | build.slave_info.update(properties) |
---|
165 | build.status = Build.IN_PROGRESS |
---|
166 | build.update(db=db) |
---|
167 | |
---|
168 | if build or builds_to_delete: |
---|
169 | db.commit() |
---|
170 | |
---|
171 | return build |
---|
172 | |
---|
173 | def match_slave(self, name, properties): |
---|
174 | """Match a build slave against available target platforms. |
---|
175 | |
---|
176 | :param name: the name of the slave |
---|
177 | :type name: `basestring` |
---|
178 | :param properties: the slave configuration |
---|
179 | :type properties: `dict` |
---|
180 | :return: the list of platforms the slave matched |
---|
181 | """ |
---|
182 | platforms = [] |
---|
183 | |
---|
184 | for config in BuildConfig.select(self.env): |
---|
185 | for platform in TargetPlatform.select(self.env, config=config.name): |
---|
186 | match = True |
---|
187 | for propname, pattern in ifilter(None, platform.rules): |
---|
188 | try: |
---|
189 | propvalue = properties.get(propname) |
---|
190 | if not propvalue or not re.match(pattern, |
---|
191 | propvalue, re.I): |
---|
192 | match = False |
---|
193 | break |
---|
194 | except re.error: |
---|
195 | self.log.error('Invalid platform matching pattern "%s"', |
---|
196 | pattern, exc_info=True) |
---|
197 | match = False |
---|
198 | break |
---|
199 | if match: |
---|
200 | self.log.debug('Slave %r matched target platform %r of ' |
---|
201 | 'build configuration %r', name, |
---|
202 | platform.name, config.name) |
---|
203 | platforms.append(platform) |
---|
204 | |
---|
205 | if not platforms: |
---|
206 | self.log.warning('Slave %r matched none of the target platforms', |
---|
207 | name) |
---|
208 | |
---|
209 | return platforms |
---|
210 | |
---|
211 | def populate(self): |
---|
212 | """Add a build for the next change on each build configuration to the |
---|
213 | queue. |
---|
214 | |
---|
215 | The next change is the latest repository check-in for which there isn't |
---|
216 | a corresponding build on each target platform. Repeatedly calling this |
---|
217 | method will eventually result in the entire change history of the build |
---|
218 | configuration being in the build queue. |
---|
219 | """ |
---|
220 | db = self.env.get_db_cnx() |
---|
221 | builds = [] |
---|
222 | |
---|
223 | for config in BuildConfig.select(self.env, db=db): |
---|
224 | platforms = [] |
---|
225 | for platform, rev, build in collect_changes(config, db=db): |
---|
226 | |
---|
227 | if not self.build_all and platform.id in platforms: |
---|
228 | # We've seen this platform already, so these are older |
---|
229 | # builds that should only be built if built_all=True |
---|
230 | self.log.debug('Ignoring older revisions for configuration ' |
---|
231 | '%r on %r', config.name, platform.name) |
---|
232 | break |
---|
233 | |
---|
234 | platforms.append(platform.id) |
---|
235 | |
---|
236 | if build is None: |
---|
237 | self.log.info('Enqueuing build of configuration "%s" at ' |
---|
238 | 'revision [%s] on %s', config.name, rev, |
---|
239 | platform.name) |
---|
240 | _repos_name, repos, _repos_path = get_repos( |
---|
241 | self.env, config.path, None) |
---|
242 | |
---|
243 | rev_time = to_timestamp(repos.get_changeset(rev).date) |
---|
244 | age = int(time.time()) - rev_time |
---|
245 | if self.stabilize_wait and age < self.stabilize_wait: |
---|
246 | self.log.info('Delaying build of revision %s until %s ' |
---|
247 | 'seconds pass. Current age is: %s ' |
---|
248 | 'seconds' % (rev, self.stabilize_wait, |
---|
249 | age)) |
---|
250 | continue |
---|
251 | |
---|
252 | build = Build(self.env, config=config.name, |
---|
253 | platform=platform.id, rev=str(rev), |
---|
254 | rev_time=rev_time) |
---|
255 | builds.append(build) |
---|
256 | |
---|
257 | for build in builds: |
---|
258 | try: |
---|
259 | build.insert(db=db) |
---|
260 | db.commit() |
---|
261 | except Exception, e: |
---|
262 | # really only want to catch IntegrityErrors raised when |
---|
263 | # a second slave attempts to add builds with the same |
---|
264 | # (config, platform, rev) as an existing build. |
---|
265 | self.log.info('Failed to insert build of configuration "%s" ' |
---|
266 | 'at revision [%s] on platform [%s]: %s', |
---|
267 | build.config, build.rev, build.platform, e) |
---|
268 | db.rollback() |
---|
269 | |
---|
270 | def reset_orphaned_builds(self): |
---|
271 | """Reset all in-progress builds to ``PENDING`` state if they've been |
---|
272 | running so long that the configured timeout has been reached. |
---|
273 | |
---|
274 | This is used to cleanup after slaves that have unexpectedly cancelled |
---|
275 | a build without notifying the master, or are for some other reason not |
---|
276 | reporting back status updates. |
---|
277 | """ |
---|
278 | if not self.timeout: |
---|
279 | # If no timeout is set, none of the in-progress builds can be |
---|
280 | # considered orphaned |
---|
281 | return |
---|
282 | |
---|
283 | db = self.env.get_db_cnx() |
---|
284 | now = int(time.time()) |
---|
285 | for build in Build.select(self.env, status=Build.IN_PROGRESS, db=db): |
---|
286 | if now - build.last_activity < self.timeout: |
---|
287 | # This build has not reached the timeout yet, assume it's still |
---|
288 | # being executed |
---|
289 | continue |
---|
290 | |
---|
291 | self.log.info('Orphaning build %d. Last activity was %s (%s)' % \ |
---|
292 | (build.id, format_datetime(build.last_activity), |
---|
293 | pretty_timedelta(build.last_activity))) |
---|
294 | |
---|
295 | build.status = Build.PENDING |
---|
296 | build.slave = None |
---|
297 | build.slave_info = {} |
---|
298 | build.started = 0 |
---|
299 | build.stopped = 0 |
---|
300 | build.last_activity = 0 |
---|
301 | for step in list(BuildStep.select(self.env, build=build.id, db=db)): |
---|
302 | step.delete(db=db) |
---|
303 | build.update(db=db) |
---|
304 | |
---|
305 | Attachment.delete_all(self.env, 'build', build.resource.id, db) |
---|
306 | db.commit() |
---|
307 | |
---|
308 | def should_delete_build(self, build, repos): |
---|
309 | config = BuildConfig.fetch(self.env, build.config) |
---|
310 | config_name = config and config.name \ |
---|
311 | or 'unknown config "%s"' % build.config |
---|
312 | |
---|
313 | platform = TargetPlatform.fetch(self.env, build.platform) |
---|
314 | # Platform may or may not exist anymore - get safe name for logging |
---|
315 | platform_name = platform and platform.name \ |
---|
316 | or 'unknown platform "%s"' % build.platform |
---|
317 | |
---|
318 | # Drop build if platform no longer exists |
---|
319 | if not platform: |
---|
320 | self.log.info('Dropping build of configuration "%s" at ' |
---|
321 | 'revision [%s] on %s because the platform no longer ' |
---|
322 | 'exists', config.name, build.rev, platform_name) |
---|
323 | return True |
---|
324 | |
---|
325 | # Ignore pending builds for deactived build configs |
---|
326 | if not (config and config.active): |
---|
327 | self.log.info('Dropping build of configuration "%s" at ' |
---|
328 | 'revision [%s] on %s because the configuration is ' |
---|
329 | 'deactivated', config_name, build.rev, platform_name) |
---|
330 | return True |
---|
331 | |
---|
332 | # Stay within the revision limits of the build config |
---|
333 | if (config.min_rev and repos.rev_older_than(build.rev, |
---|
334 | config.min_rev)) \ |
---|
335 | or (config.max_rev and repos.rev_older_than(config.max_rev, |
---|
336 | build.rev)): |
---|
337 | self.log.info('Dropping build of configuration "%s" at revision [%s] on ' |
---|
338 | '"%s" because it is outside of the revision range of the ' |
---|
339 | 'configuration', config.name, build.rev, platform_name) |
---|
340 | return True |
---|
341 | |
---|
342 | # If not 'build_all', drop if a more recent revision is available |
---|
343 | if not self.build_all and \ |
---|
344 | len(list(Build.select(self.env, config=build.config, |
---|
345 | min_rev_time=build.rev_time, platform=build.platform))) > 1: |
---|
346 | self.log.info('Dropping build of configuration "%s" at revision [%s] ' |
---|
347 | 'on "%s" because a more recent build exists', |
---|
348 | config.name, build.rev, platform_name) |
---|
349 | return True |
---|
350 | |
---|
351 | return False |
---|