Commit 35a4f4ed authored by Romain Bouvier's avatar Romain Bouvier
Browse files

Cleanup bztogl to remove gnome/fdo customization

- This will ease our own modifications
parent e65f3f25
......@@ -42,7 +42,7 @@ At a high level, `bztogl` needs to be able to do these:
look at bugs that are already closed.* Here, by "bugs that are open"
we mean bugs with status `NEW, ASSIGNED, REOPENED, NEEDINFO,
UNCONFIRMED`.
You can migrate CLOSED/RESOLVED bugs with `--resolved` switch.
You can migrate `CLOSED, RESOLVED` bugs with `--resolved` switch.
# Know what you are doing
......@@ -57,7 +57,7 @@ test project instead.
1. Create a personal project for testing
2. Get an API key
3. Run `bztogl --xfce --production`
3. Run `bztogl --production`
Each of these steps is detailed below.
......@@ -80,6 +80,7 @@ talk to Gitlab.
You need to create an "admin" account for the migration, to be able to get all
user informations (user email etc).
Get an API key at https://gitlab.xfce.org/profile/personal_access_tokens
— you can use "`bztogl`" for in the **Name** field of the application you want
**Turn on** the checkboxes for **api** and **read_user**, so that `bztogl` can
......@@ -101,10 +102,10 @@ virtualenv activated with the `activate` command form that example.
Now you can run this:
```sh
bztogl --xfce --token <your_api_token> --product Libxfce4util --target-project bugzilla-migration/libxfce4util
bztogl --token <your_api_token> --product Libxfce4util --target-project bugzilla-migration/libxfce4util
```
Be aware that Xfce production on bugzilla start with an Uppercase!
Be aware that Xfce products on bugzilla start with an Uppercase!
Using the `--production` switch will effectively do the migration ! So be sure to run it on a test `target-project`
......
......@@ -26,35 +26,6 @@ import gitlab
from . import common, milestones, template, users
NEEDINFO_LABEL = "2. Needs Information"
KEYWORD_MAP = {
"accessibility": "8. Accessibility",
"newcomers": "4. Newcomers",
"security": "1. Security"
}
COMPONENT_MAP = {
'Accessibility': '8. Accessibility',
'Backend: Broadway': 'Broadway',
'Backend: Quartz': 'MacOS',
'Backend: X11': 'X11',
'Backend: Wayland': 'Wayland',
'Backend: Win32': 'Windows',
'Documentation': '8. Developer Docs',
'Input Methods': 'Input',
'Language Bindings': 'Introspection',
'Themes': 'Theme',
'Widget: GtkComboBox': 'GtkComboBox',
'Widget: GtkEntry': 'GtkEntry',
'Widget: GtkFileChooser': '5. FileChooser',
'Widget: GtkFontChooser': 'GtkFontChooser',
'Widget: GtkMenu': 'GtkMenu',
'Widget: GtkNotebook': 'GtkNotebook',
'Widget: GtkScrolledWindow': 'GtkScrolledWindow',
'Widget: GtkSpinButton': 'GtkSpinButton',
}
def processbug(bgo, bzurl, instance, resolution, target, user_cache,
milestone_cache, bzbug, migrate_components):
print("Processing bug #%d: %s" % (bzbug.id, bzbug.summary))
......@@ -136,27 +107,6 @@ def processbug(bgo, bzurl, instance, resolution, target, user_cache,
return '\n\n'.join(converted_paragraphs)
def is_yorba_import(comment):
body = comment['text']
is_yorba = (
'Original URL: http://redmine.yorba.org/issues/' in body and
'Searchable id: yorba-bug-' in body
)
if is_yorba:
body = re.sub(r'####\n\n#', '---\n\nComment ', body)
body = re.sub(
r'\n(Original [a-zA-Z ]+: [a-zA-Z0-9.:\/ ]+)', r'\n\1 ', body
)
body = re.sub(
r'\n(Searchable id: [a-zA-Z0-9-]+)', r'\n\1 ', body
)
body = re.sub(r'\n(related to [a-zA-Z]+ - )', r'\n * \1', body)
body = re.sub(r'\n(duplicated by [a-zA-Z]+ - )', r'\n * \1', body)
body = re.sub(r'\n(blocked by [a-zA-Z]+ - )', r'\n * \1', body)
comment['text'] = body
return is_yorba
def analyze_bugzilla_comment(comment, attachment_metadata):
body = comment['text']
......@@ -203,7 +153,6 @@ def processbug(bgo, bzurl, instance, resolution, target, user_cache,
comments = bzbug.getcomments()
firstcomment = None if len(comments) < 1 else comments[0]
is_yorba = is_yorba_import(firstcomment)
desctext = None
if firstcomment and 'author' in firstcomment:
author = firstcomment['author']
......@@ -211,35 +160,17 @@ def processbug(bgo, bzurl, instance, resolution, target, user_cache,
author = firstcomment['creator']
else:
author = None
if is_yorba or author == bzbug.creator:
desctext = firstcomment['text']
if 'attachment_id' in firstcomment:
desctext += '\n' + migrate_attachment(firstcomment,
attachment_metadata)
comments = comments[1:]
description = \
template.render_issue_description(bzurl, bzbug, desctext, user_cache)
# Add a "bugzilla" label on all imported bugs
labels = ['bugzilla']
# Only set "bugzilla' label when migrating. Labels present in bz are not revelant
# if bzbug.status == 'NEEDINFO':
# labels += [NEEDINFO_LABEL]
if migrate_components:
if bzbug.component.lower() not in ('general', '.general', target.product):
labels.append('{}'.format(bzbug.component.title()))
#
# for kw in bzbug.keywords:
# if kw in KEYWORD_MAP:
# labels += [KEYWORD_MAP[kw]]
# No milestone on migration
# No milestone or label on migration
milestone = None
# bz_milestone = bzbug.target_milestone
# if bz_milestone and bz_milestone != '---':
# milestone = milestone_cache[bz_milestone]
labels = []
if user_cache[bzbug.creator]:
sudo = user_cache[bzbug.creator].id
......@@ -291,8 +222,8 @@ def processbug(bgo, bzurl, instance, resolution, target, user_cache,
issue.notes.create({
'body': gitlab_comment,
'created_at': str(comment['creation_time'])
})#, sudo=sudo)
'created_at': str(comment['creation_time'])})
#, sudo=sudo)
# Do last, so that previous actions don't all send an email
for cc_email in itertools.chain(bzbug.cc, [bzbug.creator]):
......@@ -363,10 +294,6 @@ def options():
help="project name for gitlab, like \
'username/project'. If not provided, \
$user_namespace/$bugzilla_product will be used")
parser.add_argument('--fdo', action='store_true',
help="import for freedesktop.org rather than GNOME")
parser.add_argument('--xfce', action='store_true',
help="import for xfce.org rather than GNOME")
parser.add_argument('--resolved', action='store_true',
help="import resolved bugs (default: no)")
parser.add_argument('--migrate-components', action='store_true',
......@@ -394,33 +321,14 @@ def check_if_target_project_exists(target):
def main():
args = options()
if args.fdo:
if args.production:
glurl = "https://gitlab.freedesktop.org/"
else:
glurl = "http://localhost:8080/"
bzurl = "https://bugs.freedesktop.org"
giturl = "https://anongit.freedesktop.org/git/"
instance = "freedesktop.org"
bzresolution = 'MOVED'
elif args.xfce:
if args.production:
glurl = "https://gitlab.xfce.org/"
else:
glurl = "http://localhost:8080/"
bzurl = "https://bugzilla.xfce.org"
giturl = "https://git.xfce.org/"
instance = "xfce.org"
bzresolution = 'MOVED'
if args.production:
glurl = "https://gitlab.xfce.org/"
else:
if args.production:
glurl = "https://gitlab.gnome.org/"
else:
glurl = "https://gitlab-test.gnome.org/"
bzurl = "https://bugzilla.gnome.org"
giturl = "https://git.gnome.org/browse/"
instance = "GNOME"
bzresolution = 'OBSOLETE'
glurl = "http://localhost:8080/"
bzurl = "https://bugzilla.xfce.org"
giturl = "https://git.xfce.org/"
instance = "xfce.org"
bzresolution = 'MOVED'
target = common.GitLab(glurl, giturl, args.token, args.product,
args.target_project, args.automate)
......
import argparse
import base64
import datetime
import json
import os
import re
import sys
import phabricator
from . import bt
from . import template
from . import users
from . import common
ON_WINDOWS = os.name == 'nt'
KEYWORD_MAP = {
"Pitivi tasks for newcomers": "4. Newcomers",
"translations": "8. Translation",
"titles editor": "title clips",
"bundles": "binaries"
}
MIGR_TEMPLATE = """# GitLab Migration Automatic Message
This bug has been migrated to freedesltop.org's GitLab instance and has been closed \
from further activity.
You can subscribe and participate further through the new bug through this \
link to our GitLab instance: {}.
"""
class Colors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
force_disable = False
@classmethod
def disable(cls):
cls.HEADER = ''
cls.OKBLUE = ''
cls.OKGREEN = ''
cls.WARNING = ''
cls.FAIL = ''
cls.ENDC = ''
@classmethod
def enable(cls):
if cls.force_disable:
return
cls.HEADER = '\033[95m'
cls.OKBLUE = '\033[94m'
cls.OKGREEN = '\033[92m'
cls.WARNING = '\033[93m'
cls.FAIL = '\033[91m'
cls.ENDC = '\033[0m'
class PhabGitLab(common.GitLab):
"""GitLab pabricator importer"""
def __init__(self, glurl, giturl, token, product, target_project=None,
automate=False, close_tasks=False):
super().__init__(glurl, giturl, token, product, target_project,
automate)
self.close_tasks = close_tasks
def import_from_phab(self, phab, start_at):
"""Imports project tasks from phabricator"""
if self.target_project:
projname = self.target_project.split("/")[1]
else:
projname = self.project
for _id, task in sorted(phab.tasks.items()):
if start_at and _id < start_at:
continue
description = \
template.render_issue_description(
None, task, phab.escape_markdown(
task["description"]), phab.users,
importing_address=os.path.join(phab.phabricator_uri, "T"),
bug_url_function=phab.task_url)
labels = ['phabricator']
for project in task.projects.values():
if project["name"] != projname:
# FIXME - Find a way to generically strip what should be
# stripped.
label = KEYWORD_MAP.get(project["name"], project["name"])
for to_strip in ["pitivi_", "pitivi", "ptv_", "ptv",
"Pitivi", " "]:
label = label.strip(to_strip)
labels.append(label)
if not task["title"]:
print("WARNING task %s doesn't have a title!" % _id)
continue
# Assign bug to actual account if exists
phabauthor = phab.users.get(task["authorPHID"])
if phabauthor:
author = self.find_user_by_nick(phabauthor.username)
if author:
author = author.id
else:
author = None
phabowner = phab.users.get(task["ownerPHID"])
if phabowner:
assignee = self.find_user_by_nick(phabowner.username)
else:
assignee = None
issue = self.create_issue(_id, task["title"],
description, labels,
None,
datetime.datetime.fromtimestamp(
int(task["dateCreated"])
).strftime('%Y-%m-%d %H:%M:%S')#, sudo=author
)
if assignee:
issue.assignee_id = assignee.id
print("Created %s - %s: %s" %
(_id, issue.get_id(), issue.attributes['title']))
for comment in task.comments:
emoji, action, body = ('speech_balloon', 'said',
comment["comments"])
author = phab.users[comment["authorPHID"]]
if phabowner:
sudo = self.find_user_by_nick(author.username)
if sudo:
sudo = sudo.id
else:
sudo = None
assignee = None
gitlab_comment = template.render_comment(
None, emoji, author.display_name(),
action, phab.escape_markdown(body),
"", bug_url_function=phab.task_url)
issue.notes.create({
'body': gitlab_comment,
'created_at': datetime.datetime.fromtimestamp(
int(task["dateCreated"])
).strftime('%Y-%m-%d %H:%M:%S')
})#, sudo=sudo)
state_event = 'reopen'
if task.resolved:
state_event = "close"
issue.state_event = state_event
issue.save(state_event=state_event)
if self.close_tasks:
phab.phabricator.maniphest.edit(
objectIdentifier=str(_id),
transactions=[{
"type": "comment",
"value": MIGR_TEMPLATE.format(issue.web_url)}])
phab.phabricator.maniphest.update(id=_id, status='resolved')
class Task:
def __init__(self, entry, all_projects, all_tasks):
self.entry = entry
self.projects = {}
for phid in entry["projectPHIDs"]:
self.projects[phid] = all_projects.data[phid]
self.depends_on = []
for phid in self.entry["dependsOnTaskPHIDs"]:
if phid in all_tasks:
self.depends_on.append(all_tasks[phid]["id"])
@property
def assigned_to(self):
if self.entry["ownerPHID"]:
return self.entry["ownerPHID"]
return self.entry["authorPHID"]
@property
def id(self):
return self.entry["id"]
@property
def resolved(self):
return self.entry["isClosed"]
@property
def creator(self):
return self.entry["authorPHID"]
@property
def comments(self):
return self.entry.get("comments", [])
@property
def blocks(self):
# FIXME!
return []
@property
def see_also(self):
return []
@property
def version(self):
return None
def __getitem__(self, key):
return self.entry[key]
class Phab:
FILES_REGEX = re.compile(r'\{F[0-9\(\)]+\}', re.MULTILINE)
def __init__(self, options, gitlab):
self._phabricator = None
self.arcrc = None
self.phabricator_uri = "https://phabricator.freedesktop.org/"
self.projects = options.projects
self.users = {}
self.ensure_project_phids()
self.retrieve_all_tasks()
self.gitlab = gitlab
def migrate_attachment(self, fileid):
finfo = self.phabricator.file.info(id=int(fileid))
attfile = self.phabricator.file.download(phid=finfo["phid"])
ret = self.gitlab.upload_file(
finfo["name"], base64.b64decode(attfile.response))
return ret['markdown']
def escape_markdown(self, markdown):
markdown = bt.quote_stack_traces(markdown)
# Revert possibly double quoted backtraces
markdown = re.sub(re.compile(
r'```\n```\n', re.MULTILINE), '\n```\n', markdown)
for filelink in re.findall(Phab.FILES_REGEX, markdown):
fileid = filelink.strip("{F").strip("}")
try:
markdown = markdown.replace(
filelink, self.migrate_attachment(fileid))
except phabricator.APIError:
print("WARNING: Could not migrate file: %s" % filelink)
pass
# Prevent spurious links to other GitLab issues
markdown = re.sub(r'([Cc]omment) #([0-9]+)', '\\1 \\2', markdown)
# Prevent unintended linking to issues.
markdown = re.sub(r'(\W)#([0-9]+)', '\\1# \\2', markdown)
# Link Tasks and Differentials
markdown = re.sub(
r'\b#?([TD][0-9]+)', '[\\1](%s/\\1)' % ("https://phabricator.freedesktop.org"),
markdown)
# Avoid losing new lines.
markdown = re.sub(r'([^\n])\n', '\\1 \n', markdown)
# Quote XML-like tags which would otherwise be stripped by GitLab
tags_outside_quotes = re.compile(r"""
(
^[^`]* # An initial string with no backticks
(?: # Then zero or more of...
(?:
\n```.*?\n``` # A matched pair of triple backticks
|
`[^`]+` # Or a matched pair of backticks
)
[^`]*? # Followed by a str without backticks
)*? # These are skipped before finding
)
(\<\/?[a-zA-Z0-9_="' -]*?\>) # ...an XML-like tag
""", re.VERBOSE)
nsubs = 1
while nsubs > 0:
# Matches may overlap, so we have to keep substituting until
# none left
markdown, nsubs = tags_outside_quotes.subn('\\1`\\2`', markdown)
return markdown
def task_url(self, garbage, task_id):
return os.path.join(self.phabricator_uri, "T" + task_id)
def retrieve_all_comments(self, ids, users):
all_transactions = self.phabricator.maniphest.gettasktransactions(
ids=ids)
for tid, transactions in all_transactions.items():
for transaction in sorted(transactions,
key=lambda x: x['dateCreated']):
if transaction["transactionType"] == "core:customfield":
user = self.users[transaction["authorPHID"]]
transaction["comments"] = "%s set git URI to %s" % (
user.display_name(), transaction["newValue"]
)
if transaction["comments"]:
self.tasks[int(tid)].entry.setdefault(
"comments", []).append(transaction)
def retrieve_all_users(self, usersphids):
all_users = self.phabricator.user.query(limit=99999999)
for user in all_users:
user = users.User(email=user["phid"],
real_name=user["realName"],
username=user["userName"], id=None)
self.users[user.email] = user
def retrieve_all_tasks(self):
ids = []
self.tasks = {}
users = set()
all_tasks = self.phabricator.maniphest.query(limit=9999999999, status="status-open", projectPHIDs=self.project_phids)
for task in all_tasks.items():
for phid in task[1]['projectPHIDs']:
if phid in self.project_phids:
task = task[1]
id = int(task["id"])
ids.append(id)
self.tasks[id] = Task(task, self.all_projects, all_tasks)
users.add(task["authorPHID"])
users.update(task["ccPHIDs"])
users.add(task["ownerPHID"])
break
self.retrieve_all_users(users)
self.retrieve_all_comments(ids, users)
def ensure_project_phids(self):
self.all_projects = self.phabricator.project.query(limit=9999999999, status="status-open")
self.project_phids = []
project_map = {}
for (phid, data) in self.all_projects.data.items():
project_map[data["name"].lower()] = phid
for s in data["slugs"]:
project_map[s.lower()] = phid
try:
for p in self.projects:
if p not in project_map:
print("%sProject `%s` doesn't seem to exist%s" %
(Colors.FAIL, p, Colors.ENDC))
raise
self.project_phids.append(project_map[p])
except Exception:
self.die("Failed to look up projects in Phabricator")
def setup_login_certificate(self):
token = input("""LOGIN TO PHABRICATOR
Open this page in your browser and login to Phabricator if necessary:
%s/conduit/login/
Then paste the API Token on that page below.
Paste API Token from that page and press <enter>: """ % self.phabricator_uri)
path = os.path.join(os.environ['AppData'] if ON_WINDOWS
else os.path.expanduser('~'), '.arcrc')
host = self.phabricator_uri + "/api/"
host_token = {"token": token}
try:
with open(path) as f:
arcrc = json.load(f)
if arcrc.get("hosts"):
arcrc["hosts"][host] = host_token
else:
arcrc = {
"hosts": {host: host_token}}
except (FileNotFoundError, ValueError):
arcrc = {"hosts": {host: host_token}}
with open(path, "w") as f:
print("Writing %s" % path)
json.dump(arcrc, f, indent=2)
return True
def die(self, message):
print(message, file=sys.stderr)
sys.exit(1)
@property
def phabricator(self):
if self._phabricator:
return self._phabricator
if self.arcrc:
try: