1 | # -*- coding: utf-8 -*- |
---|
2 | # |
---|
3 | # Copyright (C) 2007-2010 Edgewall Software |
---|
4 | # Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de> |
---|
5 | # All rights reserved. |
---|
6 | # |
---|
7 | # This software is licensed as described in the file COPYING, which |
---|
8 | # you should have received as part of this distribution. The terms |
---|
9 | # are also available at http://bitten.edgewall.org/wiki/License. |
---|
10 | |
---|
11 | """Automated upgrades for the Bitten database tables, and other data stored |
---|
12 | in the Trac environment. |
---|
13 | |
---|
14 | **Do not import and call directly!**""" |
---|
15 | |
---|
16 | import os |
---|
17 | import sys |
---|
18 | |
---|
19 | from trac.core import TracError |
---|
20 | from trac.db import DatabaseManager, Table, Column, Index |
---|
21 | from trac.util.text import to_unicode |
---|
22 | import codecs |
---|
23 | |
---|
24 | __docformat__ = 'restructuredtext en' |
---|
25 | |
---|
26 | # database abstraction functions |
---|
27 | |
---|
28 | def parse_scheme(env): |
---|
29 | """Retrieve the environment database scheme.""" |
---|
30 | connection_uri = DatabaseManager(env).connection_uri |
---|
31 | parts = connection_uri.split(':', 1) |
---|
32 | scheme = parts[0].lower() |
---|
33 | return scheme |
---|
34 | |
---|
35 | def update_sequence(env, db, tbl, col): |
---|
36 | """Update a sequence associated with an autoincrement column.""" |
---|
37 | # Hopefully Trac will eventually implement its own version |
---|
38 | # of this function. |
---|
39 | scheme = parse_scheme(env) |
---|
40 | if scheme == "postgres": |
---|
41 | seq = '%s_%s_seq' % (tbl, col) |
---|
42 | cursor = db.cursor() |
---|
43 | cursor.execute("SELECT setval('%s', (SELECT MAX(%s) FROM %s))" |
---|
44 | % (seq, col, tbl)) |
---|
45 | |
---|
46 | def drop_index(env, db, tbl, idx): |
---|
47 | """Drop an index associated with a table.""" |
---|
48 | # Hopefully Trac will eventually implement its own version |
---|
49 | # of this function. |
---|
50 | scheme = parse_scheme(env) |
---|
51 | cursor = db.cursor() |
---|
52 | if scheme == "mysql": |
---|
53 | cursor.execute("DROP INDEX %s ON %s" % (idx, tbl)) |
---|
54 | else: |
---|
55 | cursor.execute("DROP INDEX %s" % (idx,)) |
---|
56 | |
---|
57 | # upgrade scripts |
---|
58 | |
---|
59 | def add_log_table(env, db): |
---|
60 | """Add a table for storing the builds logs.""" |
---|
61 | INFO_LEVEL = 'I' |
---|
62 | |
---|
63 | cursor = db.cursor() |
---|
64 | |
---|
65 | build_log_schema_v3 = [ |
---|
66 | Table('bitten_log', key='id')[ |
---|
67 | Column('id', auto_increment=True), Column('build', type='int'), |
---|
68 | Column('step'), Column('type') |
---|
69 | ], |
---|
70 | Table('bitten_log_message', key=('log', 'line'))[ |
---|
71 | Column('log', type='int'), Column('line', type='int'), |
---|
72 | Column('level', size=1), Column('message') |
---|
73 | ] |
---|
74 | ] |
---|
75 | |
---|
76 | build_step_schema_v3 = [ |
---|
77 | Table('bitten_step', key=('build', 'name'))[ |
---|
78 | Column('build', type='int'), Column('name'), Column('description'), |
---|
79 | Column('status', size=1), Column('started', type='int'), |
---|
80 | Column('stopped', type='int') |
---|
81 | ] |
---|
82 | ] |
---|
83 | |
---|
84 | connector, _ = DatabaseManager(env)._get_connector() |
---|
85 | for table in build_log_schema_v3: |
---|
86 | for stmt in connector.to_sql(table): |
---|
87 | cursor.execute(stmt) |
---|
88 | |
---|
89 | update_cursor = db.cursor() |
---|
90 | cursor.execute("SELECT build,name,log FROM bitten_step " |
---|
91 | "WHERE log IS NOT NULL") |
---|
92 | for build, step, log in cursor: |
---|
93 | update_cursor.execute("INSERT INTO bitten_log (build, step) " |
---|
94 | "VALUES (%s,%s)", (build, step)) |
---|
95 | log_id = db.get_last_id(update_cursor, 'bitten_log') |
---|
96 | messages = [(log_id, line, INFO_LEVEL, msg) |
---|
97 | for line, msg in enumerate(log.splitlines())] |
---|
98 | update_cursor.executemany("INSERT INTO bitten_log_message (log, line, level, message) " |
---|
99 | "VALUES (%s, %s, %s, %s)", messages) |
---|
100 | |
---|
101 | cursor.execute("CREATE TEMPORARY TABLE old_step AS SELECT * FROM bitten_step") |
---|
102 | cursor.execute("DROP TABLE bitten_step") |
---|
103 | for table in build_step_schema_v3: |
---|
104 | for stmt in connector.to_sql(table): |
---|
105 | cursor.execute(stmt) |
---|
106 | cursor.execute("INSERT INTO bitten_step (build,name,description,status," |
---|
107 | "started,stopped) SELECT build,name,description,status," |
---|
108 | "started,stopped FROM old_step") |
---|
109 | |
---|
110 | def add_recipe_to_config(env, db): |
---|
111 | """Add a column for storing the build recipe to the build configuration |
---|
112 | table.""" |
---|
113 | cursor = db.cursor() |
---|
114 | |
---|
115 | build_config_schema_v3 = Table('bitten_config', key='name')[ |
---|
116 | Column('name'), Column('path'), Column('active', type='int'), |
---|
117 | Column('recipe'), Column('min_rev'), Column('max_rev'), |
---|
118 | Column('label'), Column('description') |
---|
119 | ] |
---|
120 | |
---|
121 | cursor.execute("CREATE TEMPORARY TABLE old_config_v2 AS " |
---|
122 | "SELECT * FROM bitten_config") |
---|
123 | cursor.execute("DROP TABLE bitten_config") |
---|
124 | |
---|
125 | connector, _ = DatabaseManager(env)._get_connector() |
---|
126 | for stmt in connector.to_sql(build_config_schema_v3): |
---|
127 | cursor.execute(stmt) |
---|
128 | |
---|
129 | cursor.execute("INSERT INTO bitten_config (name,path,active,recipe,min_rev," |
---|
130 | "max_rev,label,description) SELECT name,path,0,'',NULL," |
---|
131 | "NULL,label,description FROM old_config_v2") |
---|
132 | |
---|
133 | def add_last_activity_to_build(env, db): |
---|
134 | """Add a column for storing the last activity to the build table.""" |
---|
135 | cursor = db.cursor() |
---|
136 | |
---|
137 | build_table_schema_v12 = Table('bitten_build', key='id')[ |
---|
138 | Column('id', auto_increment=True), Column('config'), Column('rev'), |
---|
139 | Column('rev_time', type='int'), Column('platform', type='int'), |
---|
140 | Column('slave'), Column('started', type='int'), |
---|
141 | Column('stopped', type='int'), Column('status', size=1), |
---|
142 | Column('last_activity', type='int'), |
---|
143 | Index(['config', 'rev', 'platform'], unique=True) |
---|
144 | ] |
---|
145 | |
---|
146 | cursor.execute("CREATE TEMPORARY TABLE old_build_v11 AS " |
---|
147 | "SELECT * FROM bitten_build") |
---|
148 | cursor.execute("DROP TABLE bitten_build") |
---|
149 | |
---|
150 | connector, _ = DatabaseManager(env)._get_connector() |
---|
151 | for stmt in connector.to_sql(build_table_schema_v12): |
---|
152 | cursor.execute(stmt) |
---|
153 | |
---|
154 | # it's safe to make the last activity the stop time of the build |
---|
155 | cursor.execute("INSERT INTO bitten_build (id,config,rev,rev_time,platform," |
---|
156 | "slave,started,stopped,last_activity,status) " |
---|
157 | "SELECT id,config,rev,rev_time,platform," |
---|
158 | "slave,started,stopped,stopped,status FROM old_build_v11") |
---|
159 | |
---|
160 | update_sequence(env, db, 'bitten_build', 'id') |
---|
161 | |
---|
162 | def add_config_to_reports(env, db): |
---|
163 | """Add the name of the build configuration as metadata to report documents |
---|
164 | stored in the BDB XML database.""" |
---|
165 | try: |
---|
166 | from bsddb3 import db as bdb |
---|
167 | import dbxml |
---|
168 | except ImportError: |
---|
169 | return |
---|
170 | |
---|
171 | dbfile = os.path.join(env.path, 'db', 'bitten.dbxml') |
---|
172 | if not os.path.isfile(dbfile): |
---|
173 | return |
---|
174 | |
---|
175 | dbenv = bdb.DBEnv() |
---|
176 | dbenv.open(os.path.dirname(dbfile), |
---|
177 | bdb.DB_CREATE | bdb.DB_INIT_LOCK | bdb.DB_INIT_LOG | |
---|
178 | bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN, 0) |
---|
179 | |
---|
180 | mgr = dbxml.XmlManager(dbenv, 0) |
---|
181 | xtn = mgr.createTransaction() |
---|
182 | container = mgr.openContainer(dbfile, dbxml.DBXML_TRANSACTIONAL) |
---|
183 | uc = mgr.createUpdateContext() |
---|
184 | |
---|
185 | container.addIndex(xtn, '', 'config', 'node-metadata-equality-string', uc) |
---|
186 | |
---|
187 | qc = mgr.createQueryContext() |
---|
188 | for value in mgr.query(xtn, 'collection("%s")/report' % dbfile, qc): |
---|
189 | doc = value.asDocument() |
---|
190 | metaval = dbxml.XmlValue() |
---|
191 | if doc.getMetaData('', 'build', metaval): |
---|
192 | build_id = int(metaval.asNumber()) |
---|
193 | |
---|
194 | cursor = db.cursor() |
---|
195 | cursor.execute("SELECT config FROM bitten_build WHERE id=%s", (build_id,)) |
---|
196 | row = cursor.fetchone() |
---|
197 | |
---|
198 | if row: |
---|
199 | doc.setMetaData('', 'config', dbxml.XmlValue(row[0])) |
---|
200 | container.updateDocument(xtn, doc, uc) |
---|
201 | else: |
---|
202 | # an orphaned report, for whatever reason... just remove it |
---|
203 | container.deleteDocument(xtn, doc, uc) |
---|
204 | |
---|
205 | xtn.commit() |
---|
206 | container.close() |
---|
207 | dbenv.close(0) |
---|
208 | |
---|
209 | def add_order_to_log(env, db): |
---|
210 | """Add order column to log table to make sure that build logs are displayed |
---|
211 | in the order they were generated.""" |
---|
212 | cursor = db.cursor() |
---|
213 | |
---|
214 | log_table_schema_v6 = Table('bitten_log', key='id')[ |
---|
215 | Column('id', auto_increment=True), Column('build', type='int'), |
---|
216 | Column('step'), Column('generator'), Column('orderno', type='int'), |
---|
217 | Index(['build', 'step']) |
---|
218 | ] |
---|
219 | |
---|
220 | cursor.execute("CREATE TEMPORARY TABLE old_log_v5 AS " |
---|
221 | "SELECT * FROM bitten_log") |
---|
222 | cursor.execute("DROP TABLE bitten_log") |
---|
223 | |
---|
224 | connector, _ = DatabaseManager(env)._get_connector() |
---|
225 | for stmt in connector.to_sql(log_table_schema_v6): |
---|
226 | cursor.execute(stmt) |
---|
227 | |
---|
228 | cursor.execute("INSERT INTO bitten_log (id,build,step,generator,orderno) " |
---|
229 | "SELECT id,build,step,type,0 FROM old_log_v5") |
---|
230 | |
---|
231 | def add_report_tables(env, db): |
---|
232 | """Add database tables for report storage.""" |
---|
233 | cursor = db.cursor() |
---|
234 | |
---|
235 | report_schema_v6 = Table('bitten_report', key='id')[ |
---|
236 | Column('id', auto_increment=True), Column('build', type='int'), |
---|
237 | Column('step'), Column('category'), Column('generator'), |
---|
238 | Index(['build', 'step', 'category']) |
---|
239 | ] |
---|
240 | report_item_schema_v6 = Table('bitten_report_item', key=('report', 'item', 'name'))[ |
---|
241 | Column('report', type='int'), Column('item', type='int'), |
---|
242 | Column('name'), Column('value') |
---|
243 | ] |
---|
244 | |
---|
245 | connector, _ = DatabaseManager(env)._get_connector() |
---|
246 | for table in [report_schema_v6, report_item_schema_v6]: |
---|
247 | for stmt in connector.to_sql(table): |
---|
248 | cursor.execute(stmt) |
---|
249 | |
---|
250 | def xmldb_to_db(env, db): |
---|
251 | """Migrate report data from Berkeley DB XML to SQL database. |
---|
252 | |
---|
253 | Depending on the number of reports stored, this might take rather long. |
---|
254 | After the upgrade is done, the bitten.dbxml file (and any BDB XML log files) |
---|
255 | may be deleted. BDB XML is no longer used by Bitten. |
---|
256 | """ |
---|
257 | from bitten.util import xmlio |
---|
258 | try: |
---|
259 | from bsddb3 import db as bdb |
---|
260 | import dbxml |
---|
261 | except ImportError: |
---|
262 | return |
---|
263 | |
---|
264 | dbfile = os.path.join(env.path, 'db', 'bitten.dbxml') |
---|
265 | if not os.path.isfile(dbfile): |
---|
266 | return |
---|
267 | |
---|
268 | dbenv = bdb.DBEnv() |
---|
269 | dbenv.open(os.path.dirname(dbfile), |
---|
270 | bdb.DB_CREATE | bdb.DB_INIT_LOCK | bdb.DB_INIT_LOG | |
---|
271 | bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN, 0) |
---|
272 | |
---|
273 | mgr = dbxml.XmlManager(dbenv, 0) |
---|
274 | xtn = mgr.createTransaction() |
---|
275 | container = mgr.openContainer(dbfile, dbxml.DBXML_TRANSACTIONAL) |
---|
276 | |
---|
277 | def get_pylint_items(xml): |
---|
278 | for problems_elem in xml.children('problems'): |
---|
279 | for problem_elem in problems_elem.children('problem'): |
---|
280 | item = {'type': 'problem'} |
---|
281 | item.update(problem_elem.attr) |
---|
282 | yield item |
---|
283 | |
---|
284 | def get_trace_items(xml): |
---|
285 | for cov_elem in xml.children('coverage'): |
---|
286 | item = {'type': 'coverage', 'name': cov_elem.attr['module'], |
---|
287 | 'file': cov_elem.attr['file'], |
---|
288 | 'percentage': cov_elem.attr['percentage']} |
---|
289 | lines = 0 |
---|
290 | line_hits = [] |
---|
291 | for line_elem in cov_elem.children('line'): |
---|
292 | lines += 1 |
---|
293 | line_hits.append(line_elem.attr['hits']) |
---|
294 | item['lines'] = lines |
---|
295 | item['line_hits'] = ' '.join(line_hits) |
---|
296 | yield item |
---|
297 | |
---|
298 | def get_unittest_items(xml): |
---|
299 | for test_elem in xml.children('test'): |
---|
300 | item = {'type': 'test'} |
---|
301 | item.update(test_elem.attr) |
---|
302 | for child_elem in test_elem.children(): |
---|
303 | item[child_elem.name] = child_elem.gettext() |
---|
304 | yield item |
---|
305 | |
---|
306 | qc = mgr.createQueryContext() |
---|
307 | for value in mgr.query(xtn, 'collection("%s")/report' % dbfile, qc, 0): |
---|
308 | doc = value.asDocument() |
---|
309 | metaval = dbxml.XmlValue() |
---|
310 | build, step = None, None |
---|
311 | if doc.getMetaData('', 'build', metaval): |
---|
312 | build = metaval.asNumber() |
---|
313 | if doc.getMetaData('', 'step', metaval): |
---|
314 | step = metaval.asString() |
---|
315 | |
---|
316 | report_types = {'pylint': ('lint', get_pylint_items), |
---|
317 | 'trace': ('coverage', get_trace_items), |
---|
318 | 'unittest': ('test', get_unittest_items)} |
---|
319 | xml = xmlio.parse(value.asString()) |
---|
320 | report_type = xml.attr['type'] |
---|
321 | category, get_items = report_types[report_type] |
---|
322 | sys.stderr.write('.') |
---|
323 | sys.stderr.flush() |
---|
324 | |
---|
325 | items = list(get_items(xml)) |
---|
326 | |
---|
327 | cursor = db.cursor() |
---|
328 | cursor.execute("SELECT bitten_report.id FROM bitten_report " |
---|
329 | "WHERE build=%s AND step=%s AND category=%s", |
---|
330 | (build, step, category)) |
---|
331 | rows = cursor.fetchall() |
---|
332 | if rows: |
---|
333 | # Duplicate report, skip |
---|
334 | continue |
---|
335 | |
---|
336 | cursor.execute("INSERT INTO bitten_report " |
---|
337 | "(build,step,category,generator) VALUES (%s,%s,%s,%s)", |
---|
338 | (build, step, category, report_type)) |
---|
339 | id = db.get_last_id(cursor, 'bitten_report') |
---|
340 | |
---|
341 | for idx, item in enumerate(items): |
---|
342 | cursor.executemany("INSERT INTO bitten_report_item " |
---|
343 | "(report,item,name,value) VALUES (%s,%s,%s,%s)", |
---|
344 | [(id, idx, key, value) for key, value |
---|
345 | in item.items()]) |
---|
346 | |
---|
347 | sys.stderr.write('\n') |
---|
348 | sys.stderr.flush() |
---|
349 | |
---|
350 | xtn.abort() |
---|
351 | container.close() |
---|
352 | dbenv.close(0) |
---|
353 | |
---|
354 | def normalize_file_paths(env, db): |
---|
355 | """Normalize the file separator in file names in reports.""" |
---|
356 | cursor = db.cursor() |
---|
357 | cursor.execute("SELECT report,item,value FROM bitten_report_item " |
---|
358 | "WHERE name='file'") |
---|
359 | rows = cursor.fetchall() or [] |
---|
360 | for report, item, value in rows: |
---|
361 | if '\\' in value: |
---|
362 | cursor.execute("UPDATE bitten_report_item SET value=%s " |
---|
363 | "WHERE report=%s AND item=%s AND name='file'", |
---|
364 | (value.replace('\\', '/'), report, item)) |
---|
365 | |
---|
366 | def fixup_generators(env, db): |
---|
367 | """Upgrade the identifiers for the recipe commands that generated log |
---|
368 | messages and report data.""" |
---|
369 | |
---|
370 | mapping = { |
---|
371 | 'pipe': 'http://bitten.edgewall.org/tools/sh#pipe', |
---|
372 | 'make': 'http://bitten.edgewall.org/tools/c#make', |
---|
373 | 'distutils': 'http://bitten.edgewall.org/tools/python#distutils', |
---|
374 | 'exec_': 'http://bitten.edgewall.org/tools/python#exec' # Ambigious |
---|
375 | } |
---|
376 | cursor = db.cursor() |
---|
377 | cursor.execute("SELECT id,generator FROM bitten_log " |
---|
378 | "WHERE generator IN (%s)" |
---|
379 | % ','.join([repr(key) for key in mapping.keys()])) |
---|
380 | for log_id, generator in cursor: |
---|
381 | cursor.execute("UPDATE bitten_log SET generator=%s " |
---|
382 | "WHERE id=%s", (mapping[generator], log_id)) |
---|
383 | |
---|
384 | mapping = { |
---|
385 | 'unittest': 'http://bitten.edgewall.org/tools/python#unittest', |
---|
386 | 'trace': 'http://bitten.edgewall.org/tools/python#trace', |
---|
387 | 'pylint': 'http://bitten.edgewall.org/tools/python#pylint' |
---|
388 | } |
---|
389 | cursor.execute("SELECT id,generator FROM bitten_report " |
---|
390 | "WHERE generator IN (%s)" |
---|
391 | % ','.join([repr(key) for key in mapping.keys()])) |
---|
392 | for report_id, generator in cursor: |
---|
393 | cursor.execute("UPDATE bitten_report SET generator=%s " |
---|
394 | "WHERE id=%s", (mapping[generator], report_id)) |
---|
395 | |
---|
396 | def add_error_table(env, db): |
---|
397 | """Add the bitten_error table for recording step failure reasons.""" |
---|
398 | table = Table('bitten_error', key=('build', 'step', 'orderno'))[ |
---|
399 | Column('build', type='int'), Column('step'), Column('message'), |
---|
400 | Column('orderno', type='int') |
---|
401 | ] |
---|
402 | cursor = db.cursor() |
---|
403 | |
---|
404 | connector, _ = DatabaseManager(env)._get_connector() |
---|
405 | for stmt in connector.to_sql(table): |
---|
406 | cursor.execute(stmt) |
---|
407 | |
---|
408 | def add_filename_to_logs(env, db): |
---|
409 | """Add filename column to log table to save where log files are stored.""" |
---|
410 | cursor = db.cursor() |
---|
411 | |
---|
412 | build_log_schema_v9 = Table('bitten_log', key='id')[ |
---|
413 | Column('id', auto_increment=True), Column('build', type='int'), |
---|
414 | Column('step'), Column('generator'), Column('orderno', type='int'), |
---|
415 | Column('filename'), |
---|
416 | Index(['build', 'step']) |
---|
417 | ] |
---|
418 | |
---|
419 | cursor.execute("CREATE TEMPORARY TABLE old_log_v8 AS " |
---|
420 | "SELECT * FROM bitten_log") |
---|
421 | cursor.execute("DROP TABLE bitten_log") |
---|
422 | |
---|
423 | connector, _ = DatabaseManager(env)._get_connector() |
---|
424 | for stmt in connector.to_sql(build_log_schema_v9): |
---|
425 | cursor.execute(stmt) |
---|
426 | |
---|
427 | cursor.execute("INSERT INTO bitten_log (id,build,step,generator,orderno,filename) " |
---|
428 | "SELECT id,build,step,generator,orderno,'' FROM old_log_v8") |
---|
429 | |
---|
430 | def migrate_logs_to_files(env, db): |
---|
431 | """Migrates logs that are stored in the bitten_log_messages table into files.""" |
---|
432 | logs_dir = env.config.get("bitten", "logs_dir", "log/bitten") |
---|
433 | if not os.path.isabs(logs_dir): |
---|
434 | logs_dir = os.path.join(env.path, logs_dir) |
---|
435 | |
---|
436 | if os.path.exists(logs_dir): |
---|
437 | print "Bitten log folder %r already exists" % (logs_dir,) |
---|
438 | print "Upgrade cannot be performed until the existing folder is moved." |
---|
439 | print "The upgrade script will now exit with an error:\n" |
---|
440 | raise TracError("") |
---|
441 | |
---|
442 | os.makedirs(logs_dir) |
---|
443 | |
---|
444 | cursor = db.cursor() |
---|
445 | message_cursor = db.cursor() |
---|
446 | update_cursor = db.cursor() |
---|
447 | cursor.execute("SELECT id FROM bitten_log") |
---|
448 | for log_id, in cursor.fetchall(): |
---|
449 | filename = "%s.log" % (log_id,) |
---|
450 | message_cursor.execute("SELECT message, level FROM bitten_log_message WHERE log=%s ORDER BY line", (log_id,)) |
---|
451 | full_filename = os.path.join(logs_dir, filename) |
---|
452 | message_file = codecs.open(full_filename, "wb", "UTF-8") |
---|
453 | # Note: the original version of this code erroneously wrote to filename + ".level" instead of ".levels", producing unused level files |
---|
454 | level_file = codecs.open(full_filename + '.levels', "wb", "UTF-8") |
---|
455 | for message, level in message_cursor.fetchall() or []: |
---|
456 | message_file.write(to_unicode(message) + "\n") |
---|
457 | level_file.write(to_unicode(level) + "\n") |
---|
458 | message_file.close() |
---|
459 | level_file.close() |
---|
460 | update_cursor.execute("UPDATE bitten_log SET filename=%s WHERE id=%s", (filename, log_id)) |
---|
461 | env.log.info("Migrated log %s", log_id) |
---|
462 | env.log.warning("Logs have been migrated from the database to files in %s. " |
---|
463 | "Ensure permissions are set correctly on this file. " |
---|
464 | "Since we presume that the migration worked correctly, " |
---|
465 | "we are now dropping the bitten_log_message table in the database (aren't you glad you backed up)", logs_dir) |
---|
466 | cursor.close() |
---|
467 | cursor = db.cursor() |
---|
468 | cursor.execute("DROP TABLE bitten_log_message") |
---|
469 | cursor.close() |
---|
470 | env.log.warning("We have dropped the bitten_log_message table - you may want to vaccuum/compress your database to save space") |
---|
471 | |
---|
472 | def fix_log_levels_misnaming(env, db): |
---|
473 | """Renames or removes \*.log.level files created by older versions of migrate_logs_to_files.""" |
---|
474 | logs_dir = env.config.get("bitten", "logs_dir", "log/bitten") |
---|
475 | if not os.path.isabs(logs_dir): |
---|
476 | logs_dir = os.path.join(env.path, logs_dir) |
---|
477 | if not os.path.isdir(logs_dir): |
---|
478 | return |
---|
479 | |
---|
480 | rename_count = 0 |
---|
481 | rename_error_count = 0 |
---|
482 | delete_count = 0 |
---|
483 | delete_error_count = 0 |
---|
484 | |
---|
485 | for wrong_filename in os.listdir(logs_dir): |
---|
486 | if not wrong_filename.endswith('.log.level'): |
---|
487 | continue |
---|
488 | |
---|
489 | log_filename = os.path.splitext(wrong_filename)[0] |
---|
490 | right_filename = log_filename + '.levels' |
---|
491 | full_log_filename = os.path.join(logs_dir, log_filename) |
---|
492 | full_wrong_filename = os.path.join(logs_dir, wrong_filename) |
---|
493 | full_right_filename = os.path.join(logs_dir, right_filename) |
---|
494 | |
---|
495 | if not os.path.exists(full_log_filename): |
---|
496 | try: |
---|
497 | os.remove(full_wrong_filename) |
---|
498 | delete_count += 1 |
---|
499 | env.log.info("Deleted stray log level file %s", wrong_filename) |
---|
500 | except Exception, e: |
---|
501 | delete_error_count += 1 |
---|
502 | env.log.warning("Error removing stray log level file %s: %s", wrong_filename, e) |
---|
503 | else: |
---|
504 | if os.path.exists(full_right_filename): |
---|
505 | env.log.warning("Error renaming %s to %s in fix_log_levels_misnaming: new filename already exists", |
---|
506 | full_wrong_filename, full_right_filename) |
---|
507 | rename_error_count += 1 |
---|
508 | continue |
---|
509 | try: |
---|
510 | os.rename(full_wrong_filename, full_right_filename) |
---|
511 | rename_count += 1 |
---|
512 | env.log.info("Renamed incorrectly named log level file %s to %s", wrong_filename, right_filename) |
---|
513 | except Exception, e: |
---|
514 | env.log.warning("Error renaming %s to %s in fix_log_levels_misnaming: %s", full_wrong_filename, full_right_filename, e) |
---|
515 | rename_error_count += 1 |
---|
516 | |
---|
517 | env.log.info("Renamed %d incorrectly named log level files from previous migrate (%d errors)", rename_count, rename_error_count) |
---|
518 | env.log.info("Deleted %d stray log level (%d errors)", delete_count, delete_error_count) |
---|
519 | |
---|
520 | def remove_stray_log_levels_files(env, db): |
---|
521 | """Remove \*.log.levels files without a matching \*.log file (old Bitten |
---|
522 | versions did not delete .log.levels files when builds were deleted)""" |
---|
523 | logs_dir = env.config.get("bitten", "logs_dir", "log/bitten") |
---|
524 | if not os.path.isabs(logs_dir): |
---|
525 | logs_dir = os.path.join(env.path, logs_dir) |
---|
526 | if not os.path.isdir(logs_dir): |
---|
527 | return |
---|
528 | |
---|
529 | delete_count = 0 |
---|
530 | delete_error_count = 0 |
---|
531 | |
---|
532 | for filename in os.listdir(logs_dir): |
---|
533 | if not filename.endswith('.log.levels'): |
---|
534 | continue |
---|
535 | |
---|
536 | log_filename = os.path.splitext(filename)[0] |
---|
537 | full_log_filename = os.path.join(logs_dir, log_filename) |
---|
538 | full_filename = os.path.join(logs_dir, filename) |
---|
539 | |
---|
540 | if not os.path.exists(full_log_filename): |
---|
541 | try: |
---|
542 | os.remove(full_filename) |
---|
543 | delete_count += 1 |
---|
544 | env.log.info("Deleted stray log levels file %s", filename) |
---|
545 | except Exception, e: |
---|
546 | delete_error_count += 1 |
---|
547 | env.log.warning("Error removing stray log levels file %s: %s", filename, e) |
---|
548 | |
---|
549 | env.log.info("Deleted %d stray log levels (%d errors)", delete_count, delete_error_count) |
---|
550 | |
---|
551 | def recreate_rule_with_int_id(env, db): |
---|
552 | """Recreates the bitten_rule table with an integer id column rather than a text one.""" |
---|
553 | cursor = db.cursor() |
---|
554 | |
---|
555 | rule_schema_v9 = Table('bitten_rule', key=('id', 'propname'))[ |
---|
556 | Column('id', type='int'), Column('propname'), Column('pattern'), |
---|
557 | Column('orderno', type='int') |
---|
558 | ] |
---|
559 | |
---|
560 | env.log.info("Migrating bitten_rule table to integer ids") |
---|
561 | connector, _ = DatabaseManager(env)._get_connector() |
---|
562 | |
---|
563 | cursor.execute("CREATE TEMPORARY TABLE old_rule_v9 AS SELECT * FROM bitten_rule") |
---|
564 | cursor.execute("DROP TABLE bitten_rule") |
---|
565 | for stmt in connector.to_sql(rule_schema_v9): |
---|
566 | cursor.execute(stmt) |
---|
567 | cursor.execute("INSERT INTO bitten_rule (id,propname,pattern,orderno) " |
---|
568 | "SELECT %s,propname,pattern,orderno FROM old_rule_v9" % db.cast('id', 'int')) |
---|
569 | |
---|
570 | def add_config_platform_rev_index_to_build(env, db): |
---|
571 | """Adds a unique index on (config, platform, rev) to the bitten_build table. |
---|
572 | Also drops the old index on bitten_build that serves no real purpose anymore.""" |
---|
573 | # check for existing duplicates |
---|
574 | duplicates_cursor = db.cursor() |
---|
575 | build_cursor = db.cursor() |
---|
576 | |
---|
577 | duplicates_cursor.execute("SELECT config, rev, platform FROM bitten_build GROUP BY config, rev, platform HAVING COUNT(config) > 1") |
---|
578 | duplicates_exist = False |
---|
579 | for config, rev, platform in duplicates_cursor.fetchall(): |
---|
580 | if not duplicates_exist: |
---|
581 | duplicates_exist = True |
---|
582 | print "\nConfig Name, Revision, Platform :: [<list of build ids>]" |
---|
583 | print "--------------------------------------------------------" |
---|
584 | |
---|
585 | build_cursor.execute("SELECT id FROM bitten_build WHERE config='%s' AND rev='%s' AND platform='%s'" % (config, rev, platform)) |
---|
586 | build_ids = [row[0] for row in build_cursor.fetchall()] |
---|
587 | print "%s, %s, %s :: %s" % (config, rev, platform, build_ids) |
---|
588 | |
---|
589 | if duplicates_exist: |
---|
590 | print "--------------------------------------------------------\n" |
---|
591 | print "Duplicate builds found. You can obtain help on removing the" |
---|
592 | print "builds you don't want by reading the Bitten upgrade" |
---|
593 | print "documentation at:" |
---|
594 | print "http://bitten.edgewall.org/wiki/Documentation/upgrade.html" |
---|
595 | print "Upgrades cannot be performed until conflicts are resolved." |
---|
596 | print "The upgrade script will now exit with an error:\n" |
---|
597 | |
---|
598 | duplicates_cursor.close() |
---|
599 | build_cursor.close() |
---|
600 | |
---|
601 | if not duplicates_exist: |
---|
602 | cursor = db.cursor() |
---|
603 | scheme = parse_scheme(env) |
---|
604 | if scheme == "mysql": |
---|
605 | # 111 = 333 / len(columns in index) -- this is the Trac default |
---|
606 | cursor.execute("CREATE UNIQUE INDEX bitten_build_config_rev_platform_idx ON bitten_build (config(111), rev(111), platform)") |
---|
607 | else: |
---|
608 | cursor.execute("CREATE UNIQUE INDEX bitten_build_config_rev_platform_idx ON bitten_build (config,rev,platform)") |
---|
609 | drop_index(env, db, 'bitten_build', 'bitten_build_config_rev_slave_idx') |
---|
610 | else: |
---|
611 | raise TracError('') |
---|
612 | |
---|
613 | def fix_sequences(env, db): |
---|
614 | """Fixes any auto increment sequences that might have been left in an inconsistent state. |
---|
615 | |
---|
616 | Upgrade scripts for schema versions > 10 should handle sequence updates correctly themselves. |
---|
617 | """ |
---|
618 | update_sequence(env, db, 'bitten_build', 'id') |
---|
619 | update_sequence(env, db, 'bitten_log', 'id') |
---|
620 | update_sequence(env, db, 'bitten_platform', 'id') |
---|
621 | update_sequence(env, db, 'bitten_report', 'id') |
---|
622 | |
---|
623 | |
---|
624 | map = { |
---|
625 | 2: [add_log_table], |
---|
626 | 3: [add_recipe_to_config], |
---|
627 | 4: [add_config_to_reports], |
---|
628 | 5: [add_order_to_log, add_report_tables, xmldb_to_db], |
---|
629 | 6: [normalize_file_paths, fixup_generators], |
---|
630 | 7: [add_error_table], |
---|
631 | 8: [add_filename_to_logs,migrate_logs_to_files], |
---|
632 | 9: [recreate_rule_with_int_id], |
---|
633 | 10: [add_config_platform_rev_index_to_build, fix_sequences], |
---|
634 | 11: [fix_log_levels_misnaming, remove_stray_log_levels_files], |
---|
635 | 12: [add_last_activity_to_build], |
---|
636 | } |
---|