| 1 | #!/usr/bin/env python |
|---|
| 2 | |
|---|
| 3 | """ |
|---|
| 4 | Import a Bugzilla items into a Trac database. |
|---|
| 5 | |
|---|
| 6 | Requires: Trac 0.9b1 from http://trac.edgewall.org/ |
|---|
| 7 | Python 2.3 from http://www.python.org/ |
|---|
| 8 | MySQL >= 3.23 from http://www.mysql.org/ |
|---|
| 9 | |
|---|
| 10 | Thanks: Mark Rowe <mrowe@bluewire.net.nz> |
|---|
| 11 | for original TracDatabase class |
|---|
| 12 | |
|---|
| 13 | Copyright 2004, Dmitry Yusupov <dmitry_yus@yahoo.com> |
|---|
| 14 | |
|---|
| 15 | Many enhancements, Bill Soudan <bill@soudan.net> |
|---|
| 16 | Other enhancements, Florent Guillaume <fg@nuxeo.com> |
|---|
| 17 | Reworked, Jeroen Ruigrok van der Werven <asmodai@in-nomine.org> |
|---|
| 18 | |
|---|
| 19 | $Id$ |
|---|
| 20 | """ |
|---|
| 21 | |
|---|
| 22 | import re |
|---|
| 23 | |
|---|
| 24 | ### |
|---|
| 25 | ### Conversion Settings -- edit these before running if desired |
|---|
| 26 | ### |
|---|
| 27 | |
|---|
| 28 | # Bugzilla version. You can find this in Bugzilla's globals.pl file. |
|---|
| 29 | # |
|---|
| 30 | # Currently, the following bugzilla versions are known to work: |
|---|
| 31 | # 2.11 (2110), 2.16.5 (2165), 2.18.3 (2183), 2.19.1 (2191), 2.23.3 (2233) |
|---|
| 32 | # |
|---|
| 33 | # If you run this script on a version not listed here and it is successful, |
|---|
| 34 | # please file a ticket at http://trac.edgewall.org/ and assign it to |
|---|
| 35 | # jruigrok. |
|---|
| 36 | BZ_VERSION = 2180 |
|---|
| 37 | |
|---|
| 38 | # MySQL connection parameters for the Bugzilla database. These can also |
|---|
| 39 | # be specified on the command line. |
|---|
| 40 | BZ_DB = "" |
|---|
| 41 | BZ_HOST = "" |
|---|
| 42 | BZ_USER = "" |
|---|
| 43 | BZ_PASSWORD = "" |
|---|
| 44 | |
|---|
| 45 | # Path to the Trac environment. |
|---|
| 46 | TRAC_ENV = "/usr/local/trac" |
|---|
| 47 | |
|---|
| 48 | # If true, all existing Trac tickets and attachments will be removed |
|---|
| 49 | # prior to import. |
|---|
| 50 | TRAC_CLEAN = True |
|---|
| 51 | |
|---|
| 52 | # Enclose imported ticket description and comments in a {{{ }}} |
|---|
| 53 | # preformat block? This formats the text in a fixed-point font. |
|---|
| 54 | PREFORMAT_COMMENTS = False |
|---|
| 55 | |
|---|
| 56 | # Replace bug numbers in comments with #xyz |
|---|
| 57 | REPLACE_BUG_NO = False |
|---|
| 58 | |
|---|
| 59 | # Severities |
|---|
| 60 | SEVERITIES = [ |
|---|
| 61 | ("blocker", "1"), |
|---|
| 62 | ("critical", "2"), |
|---|
| 63 | ("major", "3"), |
|---|
| 64 | ("normal", "4"), |
|---|
| 65 | ("minor", "5"), |
|---|
| 66 | ("trivial", "6") |
|---|
| 67 | ] |
|---|
| 68 | |
|---|
| 69 | # Priorities |
|---|
| 70 | # If using the default Bugzilla priorities of P1 - P5, do not change anything |
|---|
| 71 | # here. |
|---|
| 72 | # If you have other priorities defined please change the P1 - P5 mapping to |
|---|
| 73 | # the order you want. You can also collapse multiple priorities on bugzilla's |
|---|
| 74 | # side into the same priority on Trac's side, simply adjust PRIORITIES_MAP. |
|---|
| 75 | PRIORITIES = [ |
|---|
| 76 | ("highest", "1"), |
|---|
| 77 | ("high", "2"), |
|---|
| 78 | ("normal", "3"), |
|---|
| 79 | ("low", "4"), |
|---|
| 80 | ("lowest", "5") |
|---|
| 81 | ] |
|---|
| 82 | |
|---|
| 83 | # Bugzilla: Trac |
|---|
| 84 | # NOTE: Use lowercase. |
|---|
| 85 | PRIORITIES_MAP = { |
|---|
| 86 | "p1": "highest", |
|---|
| 87 | "p2": "high", |
|---|
| 88 | "p3": "normal", |
|---|
| 89 | "p4": "low", |
|---|
| 90 | "p5": "lowest" |
|---|
| 91 | } |
|---|
| 92 | |
|---|
| 93 | # By default, all bugs are imported from Bugzilla. If you add a list |
|---|
| 94 | # of products here, only bugs from those products will be imported. |
|---|
| 95 | PRODUCTS = [] |
|---|
| 96 | # These Bugzilla products will be ignored during import. |
|---|
| 97 | IGNORE_PRODUCTS = [] |
|---|
| 98 | |
|---|
| 99 | # These milestones are ignored |
|---|
| 100 | IGNORE_MILESTONES = ["---"] |
|---|
| 101 | |
|---|
| 102 | # These logins are converted to these user ids |
|---|
| 103 | LOGIN_MAP = { |
|---|
| 104 | #'some.user@example.com': 'someuser', |
|---|
| 105 | } |
|---|
| 106 | |
|---|
| 107 | # These emails are removed from CC list |
|---|
| 108 | IGNORE_CC = [ |
|---|
| 109 | #'loser@example.com', |
|---|
| 110 | ] |
|---|
| 111 | |
|---|
| 112 | # The 'component' field in Trac can come either from the Product or |
|---|
| 113 | # or from the Component field of Bugzilla. COMPONENTS_FROM_PRODUCTS |
|---|
| 114 | # switches the behavior. |
|---|
| 115 | # If COMPONENTS_FROM_PRODUCTS is True: |
|---|
| 116 | # - Bugzilla Product -> Trac Component |
|---|
| 117 | # - Bugzilla Component -> Trac Keyword |
|---|
| 118 | # IF COMPONENTS_FROM_PRODUCTS is False: |
|---|
| 119 | # - Bugzilla Product -> Trac Keyword |
|---|
| 120 | # - Bugzilla Component -> Trac Component |
|---|
| 121 | COMPONENTS_FROM_PRODUCTS = False |
|---|
| 122 | |
|---|
| 123 | # If COMPONENTS_FROM_PRODUCTS is True, the default owner for each |
|---|
| 124 | # Trac component is inferred from a default Bugzilla component. |
|---|
| 125 | DEFAULT_COMPONENTS = ["default", "misc", "main"] |
|---|
| 126 | |
|---|
| 127 | # This mapping can assign keywords in the ticket entry to represent |
|---|
| 128 | # products or components (depending on COMPONENTS_FROM_PRODUCTS). |
|---|
| 129 | # The keyword will be ignored if empty. |
|---|
| 130 | KEYWORDS_MAPPING = { |
|---|
| 131 | #'Bugzilla_product_or_component': 'Keyword', |
|---|
| 132 | "default": "", |
|---|
| 133 | "misc": "", |
|---|
| 134 | } |
|---|
| 135 | |
|---|
| 136 | # If this is True, products or components are all set as keywords |
|---|
| 137 | # even if not mentionned in KEYWORDS_MAPPING. |
|---|
| 138 | MAP_ALL_KEYWORDS = True |
|---|
| 139 | |
|---|
| 140 | |
|---|
| 141 | # Bug comments that should not be imported. Each entry in list should |
|---|
| 142 | # be a regular expression. |
|---|
| 143 | IGNORE_COMMENTS = [ |
|---|
| 144 | "^Created an attachment \(id=" |
|---|
| 145 | ] |
|---|
| 146 | |
|---|
| 147 | ########################################################################### |
|---|
| 148 | ### You probably don't need to change any configuration past this line. ### |
|---|
| 149 | ########################################################################### |
|---|
| 150 | |
|---|
| 151 | # Bugzilla status to Trac status translation map. |
|---|
| 152 | # |
|---|
| 153 | # NOTE: bug activity is translated as well, which may cause bug |
|---|
| 154 | # activity to be deleted (e.g. resolved -> closed in Bugzilla |
|---|
| 155 | # would translate into closed -> closed in Trac, so we just ignore the |
|---|
| 156 | # change). |
|---|
| 157 | # |
|---|
| 158 | # There is some special magic for open in the code: if there is no |
|---|
| 159 | # Bugzilla owner, open is mapped to 'new' instead. |
|---|
| 160 | STATUS_TRANSLATE = { |
|---|
| 161 | "unconfirmed": "new", |
|---|
| 162 | "open": "assigned", |
|---|
| 163 | "resolved": "closed", |
|---|
| 164 | "verified": "closed", |
|---|
| 165 | "released": "closed" |
|---|
| 166 | } |
|---|
| 167 | |
|---|
| 168 | # Translate Bugzilla statuses into Trac keywords. This provides a way |
|---|
| 169 | # to retain the Bugzilla statuses in Trac. e.g. when a bug is marked |
|---|
| 170 | # 'verified' in Bugzilla it will be assigned a VERIFIED keyword. |
|---|
| 171 | STATUS_KEYWORDS = { |
|---|
| 172 | "verified": "VERIFIED", |
|---|
| 173 | "released": "RELEASED" |
|---|
| 174 | } |
|---|
| 175 | |
|---|
| 176 | # Some fields in Bugzilla do not have equivalents in Trac. Changes in |
|---|
| 177 | # fields listed here will not be imported into the ticket change history, |
|---|
| 178 | # otherwise you'd see changes for fields that don't exist in Trac. |
|---|
| 179 | IGNORED_ACTIVITY_FIELDS = ["everconfirmed"] |
|---|
| 180 | |
|---|
| 181 | # Regular expression and its replacement |
|---|
| 182 | BUG_NO_RE = re.compile(r"\b(bug #?)([0-9])") |
|---|
| 183 | BUG_NO_REPL = r"#\2" |
|---|
| 184 | |
|---|
| 185 | ### |
|---|
| 186 | ### Script begins here |
|---|
| 187 | ### |
|---|
| 188 | |
|---|
| 189 | import os |
|---|
| 190 | import sys |
|---|
| 191 | import string |
|---|
| 192 | import StringIO |
|---|
| 193 | |
|---|
| 194 | import MySQLdb |
|---|
| 195 | import MySQLdb.cursors |
|---|
| 196 | try: |
|---|
| 197 | from trac.env import Environment |
|---|
| 198 | except: |
|---|
| 199 | from trac.Environment import Environment |
|---|
| 200 | from trac.attachment import Attachment |
|---|
| 201 | |
|---|
| 202 | if not hasattr(sys, 'setdefaultencoding'): |
|---|
| 203 | reload(sys) |
|---|
| 204 | |
|---|
| 205 | sys.setdefaultencoding('latin1') |
|---|
| 206 | |
|---|
| 207 | # simulated Attachment class for trac.add |
|---|
| 208 | #class Attachment: |
|---|
| 209 | # def __init__(self, name, data): |
|---|
| 210 | # self.filename = name |
|---|
| 211 | # self.file = StringIO.StringIO(data.tostring()) |
|---|
| 212 | |
|---|
| 213 | # simple field translation mapping. if string not in |
|---|
| 214 | # mapping, just return string, otherwise return value |
|---|
| 215 | class FieldTranslator(dict): |
|---|
| 216 | def __getitem__(self, item): |
|---|
| 217 | if not dict.has_key(self, item): |
|---|
| 218 | return item |
|---|
| 219 | |
|---|
| 220 | return dict.__getitem__(self, item) |
|---|
| 221 | |
|---|
| 222 | statusXlator = FieldTranslator(STATUS_TRANSLATE) |
|---|
| 223 | |
|---|
| 224 | class TracDatabase(object): |
|---|
| 225 | def __init__(self, path): |
|---|
| 226 | self.env = Environment(path) |
|---|
| 227 | self._db = self.env.get_db_cnx() |
|---|
| 228 | self._db.autocommit = False |
|---|
| 229 | self.loginNameCache = {} |
|---|
| 230 | self.fieldNameCache = {} |
|---|
| 231 | |
|---|
| 232 | def db(self): |
|---|
| 233 | return self._db |
|---|
| 234 | |
|---|
| 235 | def hasTickets(self): |
|---|
| 236 | c = self.db().cursor() |
|---|
| 237 | c.execute("SELECT count(*) FROM Ticket") |
|---|
| 238 | return int(c.fetchall()[0][0]) > 0 |
|---|
| 239 | |
|---|
| 240 | def assertNoTickets(self): |
|---|
| 241 | if self.hasTickets(): |
|---|
| 242 | raise Exception("Will not modify database with existing tickets!") |
|---|
| 243 | |
|---|
| 244 | def setSeverityList(self, s): |
|---|
| 245 | """Remove all severities, set them to `s`""" |
|---|
| 246 | self.assertNoTickets() |
|---|
| 247 | |
|---|
| 248 | c = self.db().cursor() |
|---|
| 249 | c.execute("DELETE FROM enum WHERE type='severity'") |
|---|
| 250 | for value, i in s: |
|---|
| 251 | print " inserting severity '%s' - '%s'" % (value, i) |
|---|
| 252 | c.execute("""INSERT INTO enum (type, name, value) |
|---|
| 253 | VALUES (%s, %s, %s)""", |
|---|
| 254 | ("severity", value.encode('utf-8'), i)) |
|---|
| 255 | self.db().commit() |
|---|
| 256 | |
|---|
| 257 | def setPriorityList(self, s): |
|---|
| 258 | """Remove all priorities, set them to `s`""" |
|---|
| 259 | self.assertNoTickets() |
|---|
| 260 | |
|---|
| 261 | c = self.db().cursor() |
|---|
| 262 | c.execute("DELETE FROM enum WHERE type='priority'") |
|---|
| 263 | for value, i in s: |
|---|
| 264 | print " inserting priority '%s' - '%s'" % (value, i) |
|---|
| 265 | c.execute("""INSERT INTO enum (type, name, value) |
|---|
| 266 | VALUES (%s, %s, %s)""", |
|---|
| 267 | ("priority", value.encode('utf-8'), i)) |
|---|
| 268 | self.db().commit() |
|---|
| 269 | |
|---|
| 270 | |
|---|
| 271 | def setComponentList(self, l, key): |
|---|
| 272 | """Remove all components, set them to `l`""" |
|---|
| 273 | self.assertNoTickets() |
|---|
| 274 | |
|---|
| 275 | c = self.db().cursor() |
|---|
| 276 | c.execute("DELETE FROM component") |
|---|
| 277 | for comp in l: |
|---|
| 278 | print " inserting component '%s', owner '%s'" % \ |
|---|
| 279 | (comp[key], comp['owner']) |
|---|
| 280 | c.execute("INSERT INTO component (name, owner) VALUES (%s, %s)", |
|---|
| 281 | (comp[key].encode('utf-8'), |
|---|
| 282 | comp['owner'].encode('utf-8'))) |
|---|
| 283 | self.db().commit() |
|---|
| 284 | |
|---|
| 285 | def setVersionList(self, v, key): |
|---|
| 286 | """Remove all versions, set them to `v`""" |
|---|
| 287 | self.assertNoTickets() |
|---|
| 288 | |
|---|
| 289 | c = self.db().cursor() |
|---|
| 290 | c.execute("DELETE FROM version") |
|---|
| 291 | for vers in v: |
|---|
| 292 | print " inserting version '%s'" % (vers[key]) |
|---|
| 293 | c.execute("INSERT INTO version (name) VALUES (%s)", |
|---|
| 294 | (vers[key].encode('utf-8'),)) |
|---|
| 295 | self.db().commit() |
|---|
| 296 | |
|---|
| 297 | def setMilestoneList(self, m, key): |
|---|
| 298 | """Remove all milestones, set them to `m`""" |
|---|
| 299 | self.assertNoTickets() |
|---|
| 300 | |
|---|
| 301 | c = self.db().cursor() |
|---|
| 302 | c.execute("DELETE FROM milestone") |
|---|
| 303 | for ms in m: |
|---|
| 304 | milestone = ms[key] |
|---|
| 305 | print " inserting milestone '%s'" % (milestone) |
|---|
| 306 | c.execute("INSERT INTO milestone (name) VALUES (%s)", |
|---|
| 307 | (milestone.encode('utf-8'),)) |
|---|
| 308 | self.db().commit() |
|---|
| 309 | |
|---|
| 310 | def addTicket(self, id, time, changetime, component, severity, priority, |
|---|
| 311 | owner, reporter, cc, version, milestone, status, resolution, |
|---|
| 312 | summary, description, keywords): |
|---|
| 313 | c = self.db().cursor() |
|---|
| 314 | |
|---|
| 315 | desc = description.encode('utf-8') |
|---|
| 316 | type = "defect" |
|---|
| 317 | |
|---|
| 318 | if severity.lower() == "enhancement": |
|---|
| 319 | severity = "minor" |
|---|
| 320 | type = "enhancement" |
|---|
| 321 | |
|---|
| 322 | if PREFORMAT_COMMENTS: |
|---|
| 323 | desc = '{{{\n%s\n}}}' % desc |
|---|
| 324 | |
|---|
| 325 | if REPLACE_BUG_NO: |
|---|
| 326 | if BUG_NO_RE.search(desc): |
|---|
| 327 | desc = re.sub(BUG_NO_RE, BUG_NO_REPL, desc) |
|---|
| 328 | |
|---|
| 329 | if PRIORITIES_MAP.has_key(priority): |
|---|
| 330 | priority = PRIORITIES_MAP[priority] |
|---|
| 331 | |
|---|
| 332 | print " inserting ticket %s -- %s" % (id, summary) |
|---|
| 333 | |
|---|
| 334 | c.execute("""INSERT INTO ticket (id, type, time, changetime, component, |
|---|
| 335 | severity, priority, owner, reporter, |
|---|
| 336 | cc, version, milestone, status, |
|---|
| 337 | resolution, summary, description, |
|---|
| 338 | keywords) |
|---|
| 339 | VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, |
|---|
| 340 | %s, %s, %s, %s, %s, %s, %s, %s)""", |
|---|
| 341 | (id, type.encode('utf-8'), datetime2epoch(time), |
|---|
| 342 | datetime2epoch(changetime), component.encode('utf-8'), |
|---|
| 343 | severity.encode('utf-8'), priority.encode('utf-8'), owner, |
|---|
| 344 | reporter, cc, version, milestone.encode('utf-8'), |
|---|
| 345 | status.lower(), resolution, summary.encode('utf-8'), desc, |
|---|
| 346 | keywords)) |
|---|
| 347 | |
|---|
| 348 | self.db().commit() |
|---|
| 349 | return self.db().get_last_id(c, 'ticket') |
|---|
| 350 | |
|---|
| 351 | def addTicketComment(self, ticket, time, author, value): |
|---|
| 352 | comment = value.encode('utf-8') |
|---|
| 353 | |
|---|
| 354 | if PREFORMAT_COMMENTS: |
|---|
| 355 | comment = '{{{\n%s\n}}}' % comment |
|---|
| 356 | |
|---|
| 357 | if REPLACE_BUG_NO: |
|---|
| 358 | if BUG_NO_RE.search(comment): |
|---|
| 359 | comment = re.sub(BUG_NO_RE, BUG_NO_REPL, comment) |
|---|
| 360 | |
|---|
| 361 | c = self.db().cursor() |
|---|
| 362 | c.execute("""INSERT INTO ticket_change (ticket, time, author, field, |
|---|
| 363 | oldvalue, newvalue) |
|---|
| 364 | VALUES (%s, %s, %s, %s, %s, %s)""", |
|---|
| 365 | (ticket, datetime2epoch(time), author, 'comment', '', comment)) |
|---|
| 366 | self.db().commit() |
|---|
| 367 | |
|---|
| 368 | def addTicketChange(self, ticket, time, author, field, oldvalue, newvalue): |
|---|
| 369 | c = self.db().cursor() |
|---|
| 370 | |
|---|
| 371 | if field == "owner": |
|---|
| 372 | if LOGIN_MAP.has_key(oldvalue): |
|---|
| 373 | oldvalue = LOGIN_MAP[oldvalue] |
|---|
| 374 | if LOGIN_MAP.has_key(newvalue): |
|---|
| 375 | newvalue = LOGIN_MAP[newvalue] |
|---|
| 376 | |
|---|
| 377 | if field == "priority": |
|---|
| 378 | if PRIORITIES_MAP.has_key(oldvalue.lower()): |
|---|
| 379 | oldvalue = PRIORITIES_MAP[oldvalue.lower()] |
|---|
| 380 | if PRIORITIES_MAP.has_key(newvalue.lower()): |
|---|
| 381 | newvalue = PRIORITIES_MAP[newvalue.lower()] |
|---|
| 382 | |
|---|
| 383 | # Doesn't make sense if we go from highest -> highest, for example. |
|---|
| 384 | if oldvalue == newvalue: |
|---|
| 385 | return |
|---|
| 386 | |
|---|
| 387 | c.execute("""INSERT INTO ticket_change (ticket, time, author, field, |
|---|
| 388 | oldvalue, newvalue) |
|---|
| 389 | VALUES (%s, %s, %s, %s, %s, %s)""", |
|---|
| 390 | (ticket, datetime2epoch(time), author, field, |
|---|
| 391 | oldvalue.encode('utf-8'), newvalue.encode('utf-8'))) |
|---|
| 392 | self.db().commit() |
|---|
| 393 | |
|---|
| 394 | def addAttachment(self, author, a): |
|---|
| 395 | description = a['description'].encode('utf-8') |
|---|
| 396 | id = a['bug_id'] |
|---|
| 397 | filename = a['filename'].encode('utf-8') |
|---|
| 398 | filedata = StringIO.StringIO(a['thedata']) |
|---|
| 399 | filesize = len(filedata.getvalue()) |
|---|
| 400 | time = a['creation_ts'] |
|---|
| 401 | print " ->inserting attachment '%s' for ticket %s -- %s" % \ |
|---|
| 402 | (filename, id, description) |
|---|
| 403 | |
|---|
| 404 | attachment = Attachment(self.env, 'ticket', id) |
|---|
| 405 | attachment.author = author |
|---|
| 406 | attachment.description = description |
|---|
| 407 | attachment.insert(filename, filedata, filesize, datetime2epoch(time)) |
|---|
| 408 | del attachment |
|---|
| 409 | |
|---|
| 410 | def getLoginName(self, cursor, userid): |
|---|
| 411 | if userid not in self.loginNameCache: |
|---|
| 412 | cursor.execute("SELECT * FROM profiles WHERE userid = %s", (userid)) |
|---|
| 413 | loginName = cursor.fetchall() |
|---|
| 414 | |
|---|
| 415 | if loginName: |
|---|
| 416 | loginName = loginName[0]['login_name'] |
|---|
| 417 | else: |
|---|
| 418 | print """WARNING: unknown bugzilla userid %d, recording as |
|---|
| 419 | anonymous""" % (userid) |
|---|
| 420 | loginName = "anonymous" |
|---|
| 421 | |
|---|
| 422 | loginName = LOGIN_MAP.get(loginName, loginName) |
|---|
| 423 | |
|---|
| 424 | self.loginNameCache[userid] = loginName |
|---|
| 425 | |
|---|
| 426 | return self.loginNameCache[userid] |
|---|
| 427 | |
|---|
| 428 | def getFieldName(self, cursor, fieldid): |
|---|
| 429 | if fieldid not in self.fieldNa |
|---|
