| 1 | # -*- coding: utf-8 -*- |
---|
| 2 | # |
---|
| 3 | # Copyright (C) 2007-2010 Edgewall Software |
---|
| 4 | # Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de> |
---|
| 5 | # All rights reserved. |
---|
| 6 | # |
---|
| 7 | # This software is licensed as described in the file COPYING, which |
---|
| 8 | # you should have received as part of this distribution. The terms |
---|
| 9 | # are also available at http://bitten.edgewall.org/wiki/License. |
---|
| 10 | |
---|
| 11 | """Implements the scheduling of builds for a project. |
---|
| 12 | |
---|
| 13 | This module provides the functionality for scheduling builds for a specific |
---|
| 14 | Trac environment. It is used by both the build master and the web interface to |
---|
| 15 | get the list of required builds (revisions not built yet). |
---|
| 16 | |
---|
| 17 | Furthermore, the `BuildQueue` class is used by the build master to determine |
---|
| 18 | the next pending build, and to match build slaves against configured target |
---|
| 19 | platforms. |
---|
1 | 20 | """ |
---|
| 21 | |
---|
1 | 22 | from itertools import ifilter |
---|
1 | 23 | import re |
---|
1 | 24 | import time |
---|
| 25 | |
---|
1 | 26 | from trac.util.datefmt import to_timestamp |
---|
1 | 27 | from trac.util import pretty_timedelta, format_datetime |
---|
1 | 28 | from trac.attachment import Attachment |
---|
| 29 | |
---|
| 30 | |
---|
1 | 31 | from bitten.model import BuildConfig, TargetPlatform, Build, BuildStep |
---|
| 32 | |
---|
1 | 33 | __docformat__ = 'restructuredtext en' |
---|
| 34 | |
---|
| 35 | |
---|
1 | 36 | def collect_changes(repos, config, db=None): |
---|
| 37 | """Collect all changes for a build configuration that either have already |
---|
| 38 | been built, or still need to be built. |
---|
| 39 | |
---|
| 40 | This function is a generator that yields ``(platform, rev, build)`` tuples, |
---|
| 41 | where ``platform`` is a `TargetPlatform` object, ``rev`` is the identifier |
---|
| 42 | of the changeset, and ``build`` is a `Build` object or `None`. |
---|
| 43 | |
---|
| 44 | :param repos: the version control repository |
---|
| 45 | :param config: the build configuration |
---|
| 46 | :param db: a database connection (optional) |
---|
| 47 | """ |
---|
14 | 48 | env = config.env |
---|
14 | 49 | if not db: |
---|
7 | 50 | db = env.get_db_cnx() |
---|
14 | 51 | try: |
---|
14 | 52 | node = repos.get_node(config.path, config.max_rev) |
---|
0 | 53 | except Exception, e: |
---|
0 | 54 | env.log.warn('Error accessing path %r for configuration %r', |
---|
0 | 55 | config.path, config.name, exc_info=True) |
---|
0 | 56 | return |
---|
| 57 | |
---|
61 | 58 | for path, rev, chg in node.get_history(): |
---|
| 59 | |
---|
| 60 | # Don't follow moves/copies |
---|
54 | 61 | if path != repos.normalize_path(config.path): |
---|
1 | 62 | break |
---|
| 63 | |
---|
| 64 | # Stay within the limits of the build config |
---|
53 | 65 | if config.min_rev and repos.rev_older_than(rev, config.min_rev): |
---|
1 | 66 | break |
---|
52 | 67 | if config.max_rev and repos.rev_older_than(config.max_rev, rev): |
---|
1 | 68 | continue |
---|
| 69 | |
---|
| 70 | # Make sure the repository directory isn't empty at this |
---|
| 71 | # revision |
---|
51 | 72 | old_node = repos.get_node(path, rev) |
---|
51 | 73 | is_empty = True |
---|
51 | 74 | for entry in old_node.get_entries(): |
---|
50 | 75 | is_empty = False |
---|
50 | 76 | break |
---|
51 | 77 | if is_empty: |
---|
1 | 78 | continue |
---|
| 79 | |
---|
| 80 | # For every target platform, check whether there's a build |
---|
| 81 | # of this revision |
---|
107 | 82 | for platform in TargetPlatform.select(env, config.name, db=db): |
---|
62 | 83 | builds = list(Build.select(env, config.name, rev, platform.id, |
---|
62 | 84 | db=db)) |
---|
62 | 85 | if builds: |
---|
16 | 86 | build = builds[0] |
---|
16 | 87 | else: |
---|
46 | 88 | build = None |
---|
| 89 | |
---|
62 | 90 | yield platform, rev, build |
---|
| 91 | |
---|
| 92 | |
---|
2 | 93 | class BuildQueue(object): |
---|
| 94 | """Enapsulates the build queue of an environment. |
---|
| 95 | |
---|
| 96 | A build queue manages the the registration of build slaves and detection of |
---|
| 97 | repository revisions that need to be built. |
---|
1 | 98 | """ |
---|
| 99 | |
---|
1 | 100 | def __init__(self, env, build_all=False, stabilize_wait=0, timeout=0): |
---|
| 101 | """Create the build queue. |
---|
| 102 | |
---|
| 103 | :param env: the Trac environment |
---|
| 104 | :param build_all: whether older revisions should be built |
---|
| 105 | :param stabilize_wait: The time in seconds to wait before considering |
---|
| 106 | the repository stable to create a build in the queue. |
---|
| 107 | :param timeout: the time in seconds after which an in-progress build |
---|
| 108 | should be considered orphaned, and reset to pending |
---|
| 109 | state |
---|
| 110 | """ |
---|
24 | 111 | self.env = env |
---|
24 | 112 | self.log = env.log |
---|
24 | 113 | self.build_all = build_all |
---|
24 | 114 | self.stabilize_wait = stabilize_wait |
---|
24 | 115 | self.timeout = timeout |
---|
| 116 | |
---|
| 117 | # Build scheduling |
---|
| 118 | |
---|
1 | 119 | def get_build_for_slave(self, name, properties): |
---|
| 120 | """Check whether one of the pending builds can be built by the build |
---|
| 121 | slave. |
---|
| 122 | |
---|
| 123 | :param name: the name of the slave |
---|
| 124 | :type name: `basestring` |
---|
| 125 | :param properties: the slave configuration |
---|
| 126 | :type properties: `dict` |
---|
| 127 | :return: the allocated build, or `None` if no build was found |
---|
| 128 | :rtype: `Build` |
---|
| 129 | """ |
---|
5 | 130 | self.log.debug('Checking for pending builds...') |
---|
| 131 | |
---|
5 | 132 | db = self.env.get_db_cnx() |
---|
5 | 133 | repos = self.env.get_repository() |
---|
5 | 134 | assert repos, 'No "(default)" Repository: Add a repository or alias ' \ |
---|
5 | 135 | 'named "(default)" to Trac.' |
---|
| 136 | |
---|
5 | 137 | self.reset_orphaned_builds() |
---|
| 138 | |
---|
| 139 | # Iterate through pending builds by descending revision timestamp, to |
---|
| 140 | # avoid the first configuration/platform getting all the builds |
---|
7 | 141 | platforms = [p.id for p in self.match_slave(name, properties)] |
---|
5 | 142 | builds_to_delete = [] |
---|
5 | 143 | build_found = False |
---|
7 | 144 | for build in Build.select(self.env, status=Build.PENDING, db=db): |
---|
4 | 145 | if self.should_delete_build(build, repos): |
---|
2 | 146 | self.log.info('Scheduling build %d for deletion', build.id) |
---|
2 | 147 | builds_to_delete.append(build) |
---|
2 | 148 | elif build.platform in platforms: |
---|
2 | 149 | build_found = True |
---|
2 | 150 | break |
---|
5 | 151 | if not build_found: |
---|
3 | 152 | self.log.debug('No pending builds.') |
---|
3 | 153 | build = None |
---|
| 154 | |
---|
| 155 | # delete any obsolete builds |
---|
7 | 156 | for build_to_delete in builds_to_delete: |
---|
2 | 157 | build_to_delete.delete(db=db) |
---|
| 158 | |
---|
5 | 159 | if build: |
---|
2 | 160 | build.slave = name |
---|
2 | 161 | build.slave_info.update(properties) |
---|
2 | 162 | build.status = Build.IN_PROGRESS |
---|
2 | 163 | build.update(db=db) |
---|
| 164 | |
---|
5 | 165 | if build or builds_to_delete: |
---|
4 | 166 | db.commit() |
---|
| 167 | |
---|
5 | 168 | return build |
---|
| 169 | |
---|
1 | 170 | def match_slave(self, name, properties): |
---|
| 171 | """Match a build slave against available target platforms. |
---|
| 172 | |
---|
| 173 | :param name: the name of the slave |
---|
| 174 | :type name: `basestring` |
---|
| 175 | :param properties: the slave configuration |
---|
| 176 | :type properties: `dict` |
---|
| 177 | :return: the list of platforms the slave matched |
---|
| 178 | """ |
---|
12 | 179 | platforms = [] |
---|
| 180 | |
---|
22 | 181 | for config in BuildConfig.select(self.env): |
---|
19 | 182 | for platform in TargetPlatform.select(self.env, config=config.name): |
---|
9 | 183 | match = True |
---|
15 | 184 | for propname, pattern in ifilter(None, platform.rules): |
---|
9 | 185 | try: |
---|
9 | 186 | propvalue = properties.get(propname) |
---|
9 | 187 | if not propvalue or not re.match(pattern, |
---|
9 | 188 | propvalue, re.I): |
---|
2 | 189 | match = False |
---|
2 | 190 | break |
---|
1 | 191 | except re.error: |
---|
1 | 192 | self.log.error('Invalid platform matching pattern "%s"', |
---|
1 | 193 | pattern, exc_info=True) |
---|
1 | 194 | match = False |
---|
1 | 195 | break |
---|
9 | 196 | if match: |
---|
6 | 197 | self.log.debug('Slave %r matched target platform %r of ' |
---|
6 | 198 | 'build configuration %r', name, |
---|
6 | 199 | platform.name, config.name) |
---|
6 | 200 | platforms.append(platform) |
---|
| 201 | |
---|
12 | 202 | if not platforms: |
---|
6 | 203 | self.log.warning('Slave %r matched none of the target platforms', |
---|
6 | 204 | name) |
---|
| 205 | |
---|
12 | 206 | return platforms |
---|
| 207 | |
---|
1 | 208 | def populate(self): |
---|
| 209 | """Add a build for the next change on each build configuration to the |
---|
| 210 | queue. |
---|
| 211 | |
---|
| 212 | The next change is the latest repository check-in for which there isn't |
---|
| 213 | a corresponding build on each target platform. Repeatedly calling this |
---|
| 214 | method will eventually result in the entire change history of the build |
---|
| 215 | configuration being in the build queue. |
---|
| 216 | """ |
---|
12 | 217 | repos = self.env.get_repository() |
---|
12 | 218 | assert repos, 'No "(default)" Repository: Add a repository or alias ' \ |
---|
12 | 219 | 'named "(default)" to Trac.' |
---|
| 220 | |
---|
11 | 221 | db = self.env.get_db_cnx() |
---|
11 | 222 | builds = [] |
---|
| 223 | |
---|
18 | 224 | for config in BuildConfig.select(self.env, db=db): |
---|
7 | 225 | platforms = [] |
---|
32 | 226 | for platform, rev, build in collect_changes(repos, config, db): |
---|
| 227 | |
---|
29 | 228 | if not self.build_all and platform.id in platforms: |
---|
| 229 | # We've seen this platform already, so these are older |
---|
| 230 | # builds that should only be built if built_all=True |
---|
4 | 231 | self.log.debug('Ignoring older revisions for configuration ' |
---|
4 | 232 | '%r on %r', config.name, platform.name) |
---|
4 | 233 | break |
---|
| 234 | |
---|
25 | 235 | platforms.append(platform.id) |
---|
| 236 | |
---|
25 | 237 | if build is None: |
---|
9 | 238 | self.log.info('Enqueuing build of configuration "%s" at ' |
---|
9 | 239 | 'revision [%s] on %s', config.name, rev, |
---|
9 | 240 | platform.name) |
---|
| 241 | |
---|
9 | 242 | rev_time = to_timestamp(repos.get_changeset(rev).date) |
---|
9 | 243 | age = int(time.time()) - rev_time |
---|
9 | 244 | if self.stabilize_wait and age < self.stabilize_wait: |
---|
0 | 245 | self.log.info('Delaying build of revision %s until %s ' |
---|
0 | 246 | 'seconds pass. Current age is: %s ' |
---|
0 | 247 | 'seconds' % (rev, self.stabilize_wait, |
---|
0 | 248 | age)) |
---|
0 | 249 | continue |
---|
| 250 | |
---|
9 | 251 | build = Build(self.env, config=config.name, |
---|
9 | 252 | platform=platform.id, rev=str(rev), |
---|
9 | 253 | rev_time=rev_time) |
---|
9 | 254 | builds.append(build) |
---|
| 255 | |
---|
20 | 256 | for build in builds: |
---|
9 | 257 | try: |
---|
9 | 258 | build.insert(db=db) |
---|
9 | 259 | db.commit() |
---|
0 | 260 | except Exception, e: |
---|
| 261 | # really only want to catch IntegrityErrors raised when |
---|
| 262 | # a second slave attempts to add builds with the same |
---|
| 263 | # (config, platform, rev) as an existing build. |
---|
0 | 264 | self.log.info('Failed to insert build of configuration "%s" ' |
---|
0 | 265 | 'at revision [%s] on platform [%s]: %s', |
---|
0 | 266 | build.config, build.rev, build.platform, e) |
---|
0 | 267 | db.rollback() |
---|
| 268 | |
---|
1 | 269 | def reset_orphaned_builds(self): |
---|
| 270 | """Reset all in-progress builds to ``PENDING`` state if they've been |
---|
| 271 | running so long that the configured timeout has been reached. |
---|
| 272 | |
---|
| 273 | This is used to cleanup after slaves that have unexpectedly cancelled |
---|
| 274 | a build without notifying the master, or are for some other reason not |
---|
| 275 | reporting back status updates. |
---|
| 276 | """ |
---|
6 | 277 | if not self.timeout: |
---|
| 278 | # If no timeout is set, none of the in-progress builds can be |
---|
| 279 | # considered orphaned |
---|
3 | 280 | return |
---|
| 281 | |
---|
3 | 282 | db = self.env.get_db_cnx() |
---|
3 | 283 | now = int(time.time()) |
---|
5 | 284 | for build in Build.select(self.env, status=Build.IN_PROGRESS, db=db): |
---|
2 | 285 | if now - build.last_activity < self.timeout: |
---|
| 286 | # This build has not reached the timeout yet, assume it's still |
---|
| 287 | # being executed |
---|
1 | 288 | continue |
---|
| 289 | |
---|
1 | 290 | self.log.info('Orphaning build %d. Last activity was %s (%s)' % \ |
---|
1 | 291 | (build.id, format_datetime(build.last_activity), |
---|
1 | 292 | pretty_timedelta(build.last_activity))) |
---|
| 293 | |
---|
1 | 294 | build.status = Build.PENDING |
---|
1 | 295 | build.slave = None |
---|
1 | 296 | build.slave_info = {} |
---|
1 | 297 | build.started = 0 |
---|
1 | 298 | build.stopped = 0 |
---|
1 | 299 | build.last_activity = 0 |
---|
1 | 300 | for step in list(BuildStep.select(self.env, build=build.id, db=db)): |
---|
0 | 301 | step.delete(db=db) |
---|
1 | 302 | build.update(db=db) |
---|
| 303 | |
---|
1 | 304 | Attachment.delete_all(self.env, 'build', build.resource.id, db) |
---|
3 | 305 | db.commit() |
---|
| 306 | |
---|
1 | 307 | def should_delete_build(self, build, repos): |
---|
10 | 308 | config = BuildConfig.fetch(self.env, build.config) |
---|
10 | 309 | config_name = config and config.name \ |
---|
1 | 310 | or 'unknown config "%s"' % build.config |
---|
| 311 | |
---|
10 | 312 | platform = TargetPlatform.fetch(self.env, build.platform) |
---|
| 313 | # Platform may or may not exist anymore - get safe name for logging |
---|
10 | 314 | platform_name = platform and platform.name \ |
---|
2 | 315 | or 'unknown platform "%s"' % build.platform |
---|
| 316 | |
---|
| 317 | # Drop build if platform no longer exists |
---|
10 | 318 | if not platform: |
---|
2 | 319 | self.log.info('Dropping build of configuration "%s" at ' |
---|
2 | 320 | 'revision [%s] on %s because the platform no longer ' |
---|
2 | 321 | 'exists', config.name, build.rev, platform_name) |
---|
2 | 322 | return True |
---|
| 323 | |
---|
| 324 | # Ignore pending builds for deactived build configs |
---|
8 | 325 | if not (config and config.active): |
---|
3 | 326 | self.log.info('Dropping build of configuration "%s" at ' |
---|
3 | 327 | 'revision [%s] on %s because the configuration is ' |
---|
3 | 328 | 'deactivated', config_name, build.rev, platform_name) |
---|
3 | 329 | return True |
---|
| 330 | |
---|
| 331 | # Stay within the revision limits of the build config |
---|
5 | 332 | if (config.min_rev and repos.rev_older_than(build.rev, |
---|
2 | 333 | config.min_rev)) \ |
---|
4 | 334 | or (config.max_rev and repos.rev_older_than(config.max_rev, |
---|
1 | 335 | build.rev)): |
---|
2 | 336 | self.log.info('Dropping build of configuration "%s" at revision [%s] on ' |
---|
2 | 337 | '"%s" because it is outside of the revision range of the ' |
---|
2 | 338 | 'configuration', config.name, build.rev, platform_name) |
---|
2 | 339 | return True |
---|
| 340 | |
---|
| 341 | # If not 'build_all', drop if a more recent revision is available |
---|
3 | 342 | if not self.build_all and \ |
---|
3 | 343 | len(list(Build.select(self.env, config=build.config, |
---|
3 | 344 | min_rev_time=build.rev_time, platform=build.platform))) > 1: |
---|
1 | 345 | self.log.info('Dropping build of configuration "%s" at revision [%s] ' |
---|
1 | 346 | 'on "%s" because a more recent build exists', |
---|
1 | 347 | config.name, build.rev, platform_name) |
---|
1 | 348 | return True |
---|
| 349 | |
---|
2 | 350 | return False |
---|