diff --git a/README.md b/README.md
index 7b89d294d78f36eafdecbe62f489a1a648b80e4c..e5bce036bd6b23be7e8770a37aa30828bce1cdb1 100644
--- a/README.md
+++ b/README.md
@@ -42,7 +42,7 @@ At a high level, `bztogl` needs to be able to do these:
look at bugs that are already closed.* Here, by "bugs that are open"
we mean bugs with status `NEW, ASSIGNED, REOPENED, NEEDINFO,
UNCONFIRMED`.
-You can migrate CLOSED/RESOLVED bugs with `--resolved` switch.
+You can migrate `CLOSED, RESOLVED` bugs with `--resolved` switch.
# Know what you are doing
@@ -57,7 +57,7 @@ test project instead.
1. Create a personal project for testing
2. Get an API key
-3. Run `bztogl --xfce --production`
+3. Run `bztogl --production`
Each of these steps is detailed below.
@@ -80,6 +80,7 @@ talk to Gitlab.
You need to create an "admin" account for the migration, to be able to get all
user informations (user email etc).
+
Get an API key at https://gitlab.xfce.org/profile/personal_access_tokens
— you can use "`bztogl`" for in the **Name** field of the application you want
**Turn on** the checkboxes for **api** and **read_user**, so that `bztogl` can
@@ -101,10 +102,10 @@ virtualenv activated with the `activate` command form that example.
Now you can run this:
```sh
-bztogl --xfce --token <your_api_token> --product Libxfce4util --target-project bugzilla-migration/libxfce4util
+bztogl --token <your_api_token> --product Libxfce4util --target-project bugzilla-migration/libxfce4util
```
-Be aware that Xfce production on bugzilla start with an Uppercase!
+Be aware that Xfce products on bugzilla start with an Uppercase!
Using the `--production` switch will effectively do the migration ! So be sure to run it on a test `target-project`
diff --git a/bztogl/bztogl.py b/bztogl/bztogl.py
index 77cb76fbf97406b05815f022df2ee5cb197e410c..a2ca5cb9ae80c6d0d067a6701c81bcdbf72d800b 100644
--- a/bztogl/bztogl.py
+++ b/bztogl/bztogl.py
@@ -26,35 +26,6 @@ import gitlab
from . import common, milestones, template, users
-NEEDINFO_LABEL = "2. Needs Information"
-KEYWORD_MAP = {
- "accessibility": "8. Accessibility",
- "newcomers": "4. Newcomers",
- "security": "1. Security"
-}
-
-COMPONENT_MAP = {
- 'Accessibility': '8. Accessibility',
- 'Backend: Broadway': 'Broadway',
- 'Backend: Quartz': 'MacOS',
- 'Backend: X11': 'X11',
- 'Backend: Wayland': 'Wayland',
- 'Backend: Win32': 'Windows',
- 'Documentation': '8. Developer Docs',
- 'Input Methods': 'Input',
- 'Language Bindings': 'Introspection',
- 'Themes': 'Theme',
- 'Widget: GtkComboBox': 'GtkComboBox',
- 'Widget: GtkEntry': 'GtkEntry',
- 'Widget: GtkFileChooser': '5. FileChooser',
- 'Widget: GtkFontChooser': 'GtkFontChooser',
- 'Widget: GtkMenu': 'GtkMenu',
- 'Widget: GtkNotebook': 'GtkNotebook',
- 'Widget: GtkScrolledWindow': 'GtkScrolledWindow',
- 'Widget: GtkSpinButton': 'GtkSpinButton',
-}
-
-
def processbug(bgo, bzurl, instance, resolution, target, user_cache,
milestone_cache, bzbug, migrate_components):
print("Processing bug #%d: %s" % (bzbug.id, bzbug.summary))
@@ -136,27 +107,6 @@ def processbug(bgo, bzurl, instance, resolution, target, user_cache,
return '\n\n'.join(converted_paragraphs)
- def is_yorba_import(comment):
- body = comment['text']
- is_yorba = (
- 'Original URL: http://redmine.yorba.org/issues/' in body and
- 'Searchable id: yorba-bug-' in body
- )
- if is_yorba:
- body = re.sub(r'####\n\n#', '---\n\nComment ', body)
- body = re.sub(
- r'\n(Original [a-zA-Z ]+: [a-zA-Z0-9.:\/ ]+)', r'\n\1 ', body
- )
- body = re.sub(
- r'\n(Searchable id: [a-zA-Z0-9-]+)', r'\n\1 ', body
- )
- body = re.sub(r'\n(related to [a-zA-Z]+ - )', r'\n * \1', body)
- body = re.sub(r'\n(duplicated by [a-zA-Z]+ - )', r'\n * \1', body)
- body = re.sub(r'\n(blocked by [a-zA-Z]+ - )', r'\n * \1', body)
- comment['text'] = body
-
- return is_yorba
-
def analyze_bugzilla_comment(comment, attachment_metadata):
body = comment['text']
@@ -203,7 +153,6 @@ def processbug(bgo, bzurl, instance, resolution, target, user_cache,
comments = bzbug.getcomments()
firstcomment = None if len(comments) < 1 else comments[0]
- is_yorba = is_yorba_import(firstcomment)
desctext = None
if firstcomment and 'author' in firstcomment:
author = firstcomment['author']
@@ -211,35 +160,17 @@ def processbug(bgo, bzurl, instance, resolution, target, user_cache,
author = firstcomment['creator']
else:
author = None
- if is_yorba or author == bzbug.creator:
- desctext = firstcomment['text']
- if 'attachment_id' in firstcomment:
- desctext += '\n' + migrate_attachment(firstcomment,
- attachment_metadata)
- comments = comments[1:]
description = \
template.render_issue_description(bzurl, bzbug, desctext, user_cache)
- # Add a "bugzilla" label on all imported bugs
- labels = ['bugzilla']
-
-# Only set "bugzilla' label when migrating. Labels present in bz are not revelant
-# if bzbug.status == 'NEEDINFO':
-# labels += [NEEDINFO_LABEL]
if migrate_components:
if bzbug.component.lower() not in ('general', '.general', target.product):
labels.append('{}'.format(bzbug.component.title()))
-#
-# for kw in bzbug.keywords:
-# if kw in KEYWORD_MAP:
-# labels += [KEYWORD_MAP[kw]]
-# No milestone on migration
+ # No milestone or label on migration
milestone = None
-# bz_milestone = bzbug.target_milestone
-# if bz_milestone and bz_milestone != '---':
-# milestone = milestone_cache[bz_milestone]
+ labels = []
if user_cache[bzbug.creator]:
sudo = user_cache[bzbug.creator].id
@@ -291,8 +222,8 @@ def processbug(bgo, bzurl, instance, resolution, target, user_cache,
issue.notes.create({
'body': gitlab_comment,
- 'created_at': str(comment['creation_time'])
- })#, sudo=sudo)
+ 'created_at': str(comment['creation_time'])})
+ #, sudo=sudo)
# Do last, so that previous actions don't all send an email
for cc_email in itertools.chain(bzbug.cc, [bzbug.creator]):
@@ -363,10 +294,6 @@ def options():
help="project name for gitlab, like \
'username/project'. If not provided, \
$user_namespace/$bugzilla_product will be used")
- parser.add_argument('--fdo', action='store_true',
- help="import for freedesktop.org rather than GNOME")
- parser.add_argument('--xfce', action='store_true',
- help="import for xfce.org rather than GNOME")
parser.add_argument('--resolved', action='store_true',
help="import resolved bugs (default: no)")
parser.add_argument('--migrate-components', action='store_true',
@@ -394,33 +321,14 @@ def check_if_target_project_exists(target):
def main():
args = options()
- if args.fdo:
- if args.production:
- glurl = "https://gitlab.freedesktop.org/"
- else:
- glurl = "http://localhost:8080/"
- bzurl = "https://bugs.freedesktop.org"
- giturl = "https://anongit.freedesktop.org/git/"
- instance = "freedesktop.org"
- bzresolution = 'MOVED'
- elif args.xfce:
- if args.production:
- glurl = "https://gitlab.xfce.org/"
- else:
- glurl = "http://localhost:8080/"
- bzurl = "https://bugzilla.xfce.org"
- giturl = "https://git.xfce.org/"
- instance = "xfce.org"
- bzresolution = 'MOVED'
+ if args.production:
+ glurl = "https://gitlab.xfce.org/"
else:
- if args.production:
- glurl = "https://gitlab.gnome.org/"
- else:
- glurl = "https://gitlab-test.gnome.org/"
- bzurl = "https://bugzilla.gnome.org"
- giturl = "https://git.gnome.org/browse/"
- instance = "GNOME"
- bzresolution = 'OBSOLETE'
+ glurl = "http://localhost:8080/"
+ bzurl = "https://bugzilla.xfce.org"
+ giturl = "https://git.xfce.org/"
+ instance = "xfce.org"
+ bzresolution = 'MOVED'
target = common.GitLab(glurl, giturl, args.token, args.product,
args.target_project, args.automate)
diff --git a/bztogl/phabtogl.py b/bztogl/phabtogl.py
deleted file mode 100755
index 3b5111b75f6aa814d8d837210877e9e8f35c45d0..0000000000000000000000000000000000000000
--- a/bztogl/phabtogl.py
+++ /dev/null
@@ -1,518 +0,0 @@
-import argparse
-import base64
-import datetime
-import json
-import os
-import re
-import sys
-
-import phabricator
-
-from . import bt
-from . import template
-from . import users
-from . import common
-
-ON_WINDOWS = os.name == 'nt'
-
-KEYWORD_MAP = {
- "Pitivi tasks for newcomers": "4. Newcomers",
- "translations": "8. Translation",
- "titles editor": "title clips",
- "bundles": "binaries"
-}
-
-
-MIGR_TEMPLATE = """# GitLab Migration Automatic Message
-
-This bug has been migrated to freedesltop.org's GitLab instance and has been closed \
-from further activity.
-
-You can subscribe and participate further through the new bug through this \
-link to our GitLab instance: {}.
-"""
-
-
-class Colors:
- HEADER = '\033[95m'
- OKBLUE = '\033[94m'
- OKGREEN = '\033[92m'
- WARNING = '\033[93m'
- FAIL = '\033[91m'
- ENDC = '\033[0m'
-
- force_disable = False
-
- @classmethod
- def disable(cls):
- cls.HEADER = ''
- cls.OKBLUE = ''
- cls.OKGREEN = ''
- cls.WARNING = ''
- cls.FAIL = ''
- cls.ENDC = ''
-
- @classmethod
- def enable(cls):
- if cls.force_disable:
- return
-
- cls.HEADER = '\033[95m'
- cls.OKBLUE = '\033[94m'
- cls.OKGREEN = '\033[92m'
- cls.WARNING = '\033[93m'
- cls.FAIL = '\033[91m'
- cls.ENDC = '\033[0m'
-
-
-class PhabGitLab(common.GitLab):
- """GitLab pabricator importer"""
-
- def __init__(self, glurl, giturl, token, product, target_project=None,
- automate=False, close_tasks=False):
- super().__init__(glurl, giturl, token, product, target_project,
- automate)
- self.close_tasks = close_tasks
-
- def import_from_phab(self, phab, start_at):
- """Imports project tasks from phabricator"""
-
- if self.target_project:
- projname = self.target_project.split("/")[1]
- else:
- projname = self.project
-
- for _id, task in sorted(phab.tasks.items()):
- if start_at and _id < start_at:
- continue
-
- description = \
- template.render_issue_description(
- None, task, phab.escape_markdown(
- task["description"]), phab.users,
- importing_address=os.path.join(phab.phabricator_uri, "T"),
- bug_url_function=phab.task_url)
-
- labels = ['phabricator']
- for project in task.projects.values():
- if project["name"] != projname:
- # FIXME - Find a way to generically strip what should be
- # stripped.
- label = KEYWORD_MAP.get(project["name"], project["name"])
- for to_strip in ["pitivi_", "pitivi", "ptv_", "ptv",
- "Pitivi", " "]:
- label = label.strip(to_strip)
- labels.append(label)
-
- if not task["title"]:
- print("WARNING task %s doesn't have a title!" % _id)
- continue
-
- # Assign bug to actual account if exists
- phabauthor = phab.users.get(task["authorPHID"])
- if phabauthor:
- author = self.find_user_by_nick(phabauthor.username)
- if author:
- author = author.id
- else:
- author = None
-
- phabowner = phab.users.get(task["ownerPHID"])
- if phabowner:
- assignee = self.find_user_by_nick(phabowner.username)
- else:
- assignee = None
-
- issue = self.create_issue(_id, task["title"],
- description, labels,
- None,
- datetime.datetime.fromtimestamp(
- int(task["dateCreated"])
- ).strftime('%Y-%m-%d %H:%M:%S')#, sudo=author
- )
-
- if assignee:
- issue.assignee_id = assignee.id
-
- print("Created %s - %s: %s" %
- (_id, issue.get_id(), issue.attributes['title']))
-
- for comment in task.comments:
- emoji, action, body = ('speech_balloon', 'said',
- comment["comments"])
- author = phab.users[comment["authorPHID"]]
- if phabowner:
- sudo = self.find_user_by_nick(author.username)
- if sudo:
- sudo = sudo.id
- else:
- sudo = None
- assignee = None
- gitlab_comment = template.render_comment(
- None, emoji, author.display_name(),
- action, phab.escape_markdown(body),
- "", bug_url_function=phab.task_url)
-
- issue.notes.create({
- 'body': gitlab_comment,
- 'created_at': datetime.datetime.fromtimestamp(
- int(task["dateCreated"])
- ).strftime('%Y-%m-%d %H:%M:%S')
- })#, sudo=sudo)
-
- state_event = 'reopen'
- if task.resolved:
- state_event = "close"
- issue.state_event = state_event
-
- issue.save(state_event=state_event)
-
- if self.close_tasks:
- phab.phabricator.maniphest.edit(
- objectIdentifier=str(_id),
- transactions=[{
- "type": "comment",
- "value": MIGR_TEMPLATE.format(issue.web_url)}])
- phab.phabricator.maniphest.update(id=_id, status='resolved')
-
-
-class Task:
- def __init__(self, entry, all_projects, all_tasks):
- self.entry = entry
- self.projects = {}
- for phid in entry["projectPHIDs"]:
- self.projects[phid] = all_projects.data[phid]
- self.depends_on = []
- for phid in self.entry["dependsOnTaskPHIDs"]:
- if phid in all_tasks:
- self.depends_on.append(all_tasks[phid]["id"])
-
- @property
- def assigned_to(self):
- if self.entry["ownerPHID"]:
- return self.entry["ownerPHID"]
-
- return self.entry["authorPHID"]
-
- @property
- def id(self):
- return self.entry["id"]
-
- @property
- def resolved(self):
- return self.entry["isClosed"]
-
- @property
- def creator(self):
- return self.entry["authorPHID"]
-
- @property
- def comments(self):
- return self.entry.get("comments", [])
-
- @property
- def blocks(self):
- # FIXME!
- return []
-
- @property
- def see_also(self):
- return []
-
- @property
- def version(self):
- return None
-
- def __getitem__(self, key):
- return self.entry[key]
-
-
-class Phab:
-
- FILES_REGEX = re.compile(r'\{F[0-9\(\)]+\}', re.MULTILINE)
-
- def __init__(self, options, gitlab):
- self._phabricator = None
- self.arcrc = None
- self.phabricator_uri = "https://phabricator.freedesktop.org/"
- self.projects = options.projects
-
- self.users = {}
- self.ensure_project_phids()
- self.retrieve_all_tasks()
- self.gitlab = gitlab
-
- def migrate_attachment(self, fileid):
- finfo = self.phabricator.file.info(id=int(fileid))
- attfile = self.phabricator.file.download(phid=finfo["phid"])
- ret = self.gitlab.upload_file(
- finfo["name"], base64.b64decode(attfile.response))
-
- return ret['markdown']
-
- def escape_markdown(self, markdown):
- markdown = bt.quote_stack_traces(markdown)
-
- # Revert possibly double quoted backtraces
- markdown = re.sub(re.compile(
- r'```\n```\n', re.MULTILINE), '\n```\n', markdown)
-
- for filelink in re.findall(Phab.FILES_REGEX, markdown):
- fileid = filelink.strip("{F").strip("}")
- try:
- markdown = markdown.replace(
- filelink, self.migrate_attachment(fileid))
- except phabricator.APIError:
- print("WARNING: Could not migrate file: %s" % filelink)
- pass
-
- # Prevent spurious links to other GitLab issues
- markdown = re.sub(r'([Cc]omment) #([0-9]+)', '\\1 \\2', markdown)
-
- # Prevent unintended linking to issues.
- markdown = re.sub(r'(\W)#([0-9]+)', '\\1# \\2', markdown)
-
- # Link Tasks and Differentials
- markdown = re.sub(
- r'\b#?([TD][0-9]+)', '[\\1](%s/\\1)' % ("https://phabricator.freedesktop.org"),
- markdown)
-
- # Avoid losing new lines.
- markdown = re.sub(r'([^\n])\n', '\\1 \n', markdown)
-
- # Quote XML-like tags which would otherwise be stripped by GitLab
- tags_outside_quotes = re.compile(r"""
- (
- ^[^`]* # An initial string with no backticks
- (?: # Then zero or more of...
- (?:
- \n```.*?\n``` # A matched pair of triple backticks
- |
- `[^`]+` # Or a matched pair of backticks
- )
- [^`]*? # Followed by a str without backticks
- )*? # These are skipped before finding
- )
- (\<\/?[a-zA-Z0-9_="' -]*?\>) # ...an XML-like tag
- """, re.VERBOSE)
-
- nsubs = 1
- while nsubs > 0:
- # Matches may overlap, so we have to keep substituting until
- # none left
- markdown, nsubs = tags_outside_quotes.subn('\\1`\\2`', markdown)
-
- return markdown
-
- def task_url(self, garbage, task_id):
- return os.path.join(self.phabricator_uri, "T" + task_id)
-
- def retrieve_all_comments(self, ids, users):
- all_transactions = self.phabricator.maniphest.gettasktransactions(
- ids=ids)
- for tid, transactions in all_transactions.items():
- for transaction in sorted(transactions,
- key=lambda x: x['dateCreated']):
- if transaction["transactionType"] == "core:customfield":
- user = self.users[transaction["authorPHID"]]
- transaction["comments"] = "%s set git URI to %s" % (
- user.display_name(), transaction["newValue"]
- )
-
- if transaction["comments"]:
- self.tasks[int(tid)].entry.setdefault(
- "comments", []).append(transaction)
-
- def retrieve_all_users(self, usersphids):
- all_users = self.phabricator.user.query(limit=99999999)
- for user in all_users:
- user = users.User(email=user["phid"],
- real_name=user["realName"],
- username=user["userName"], id=None)
- self.users[user.email] = user
-
- def retrieve_all_tasks(self):
- ids = []
-
- self.tasks = {}
- users = set()
- all_tasks = self.phabricator.maniphest.query(limit=9999999999, status="status-open", projectPHIDs=self.project_phids)
- for task in all_tasks.items():
- for phid in task[1]['projectPHIDs']:
- if phid in self.project_phids:
- task = task[1]
- id = int(task["id"])
- ids.append(id)
- self.tasks[id] = Task(task, self.all_projects, all_tasks)
- users.add(task["authorPHID"])
- users.update(task["ccPHIDs"])
- users.add(task["ownerPHID"])
-
- break
-
- self.retrieve_all_users(users)
- self.retrieve_all_comments(ids, users)
-
- def ensure_project_phids(self):
- self.all_projects = self.phabricator.project.query(limit=9999999999, status="status-open")
-
- self.project_phids = []
- project_map = {}
- for (phid, data) in self.all_projects.data.items():
- project_map[data["name"].lower()] = phid
- for s in data["slugs"]:
- project_map[s.lower()] = phid
-
- try:
- for p in self.projects:
- if p not in project_map:
- print("%sProject `%s` doesn't seem to exist%s" %
- (Colors.FAIL, p, Colors.ENDC))
- raise
- self.project_phids.append(project_map[p])
- except Exception:
- self.die("Failed to look up projects in Phabricator")
-
- def setup_login_certificate(self):
- token = input("""LOGIN TO PHABRICATOR
-Open this page in your browser and login to Phabricator if necessary:
-
-%s/conduit/login/
-
-Then paste the API Token on that page below.
-
-Paste API Token from that page and press <enter>: """ % self.phabricator_uri)
- path = os.path.join(os.environ['AppData'] if ON_WINDOWS
- else os.path.expanduser('~'), '.arcrc')
-
- host = self.phabricator_uri + "/api/"
- host_token = {"token": token}
- try:
- with open(path) as f:
- arcrc = json.load(f)
-
- if arcrc.get("hosts"):
- arcrc["hosts"][host] = host_token
- else:
- arcrc = {
- "hosts": {host: host_token}}
-
- except (FileNotFoundError, ValueError):
- arcrc = {"hosts": {host: host_token}}
-
- with open(path, "w") as f:
- print("Writing %s" % path)
- json.dump(arcrc, f, indent=2)
-
- return True
-
- def die(self, message):
- print(message, file=sys.stderr)
- sys.exit(1)
-
- @property
- def phabricator(self):
- if self._phabricator:
- return self._phabricator
-
- if self.arcrc:
- try:
- with open(self.arcrc) as f:
- phabricator.ARCRC.update(json.load(f))
- except FileNotFoundError:
- self.die("Failed to load a given arcrc file, %s" % self.arcrc)
-
- needs_credential = False
- try:
- host = self.phabricator_uri + "/api/"
- self._phabricator = phabricator.Phabricator(timeout=120, host=host)
-
- if not self.phabricator.token and not self.phabricator.certificate:
- needs_credential = True
-
- # FIXME, workaround
- # https://github.com/disqus/python-phabricator/issues/37
- self._phabricator.differential.creatediff.api.interface[
- "differential"]["creatediff"]["required"]["changes"] = dict
- except phabricator.ConfigurationError:
- needs_credential = True
-
- if needs_credential:
- if self.setup_login_certificate():
- self.die("Try again now that the login certificate has been"
- " added")
- else:
- self.die("Please setup login certificate before trying again")
-
- return self._phabricator
-
-
-def options():
- parser = argparse.ArgumentParser(
- description="Phabricator migration helper for bugzilla.gnome.org "
- "products")
- parser = argparse.ArgumentParser(
- description="Bugzilla migration helper for bugzilla.gnome.org "
- "products")
- parser.add_argument('--production', action='store_true',
- help="target production (gitlab.gnome.org) instead \
- of testing (gitlab-test.gnome.org)")
- parser.add_argument('--recreate', action='store_true',
- help="remove the project at GitLab if it exists and \
- import the project from the original repository")
- parser.add_argument('--only-import', action='store_true',
- help="only import the module, no migration of issues")
- parser.add_argument('--automate', action='store_true',
- help="don't wait on user input and answer \'Y\' (yes) \
- to any question")
- parser.add_argument('--token', help="gitlab token API", required=True)
- parser.add_argument(
- '--close-tasks', help="Close phabricator tasks", action='store_true')
- parser.add_argument('--project', help="phab project name", dest="projects",
- required=True, action="append")
- parser.add_argument('--target-project', metavar="USERNAME/PROJECT",
- help="project name for gitlab, like \
- 'username/project'. If not provided, \
- $user_namespace/$bugzilla_product will be used")
- parser.add_argument('--start-at',
- help="The ID of the first task to import",
- type=int)
- return parser.parse_args()
-
-
-def check_if_target_project_exists(target):
- try:
- target.get_project()
- except Exception as e:
- print("ERROR: Could not access the project `{}` - are you sure \
- it exists?".format(target.target_project))
- print("You can use the --target-project=username/project option if \
- the project name\n\is different from the Bugzilla \
- product name.")
- exit(1)
-
-
-def main():
- args = options()
-
- target = PhabGitLab("https://gitlab.freedesktop.org/",
- "https://cgit.freedesktop.org/",
- args.token, args.projects[0],
- args.target_project,
- args.automate, args.close_tasks)
-
- target.connect()
- if not args.recreate and args.target_project is not None:
- check_if_target_project_exists(target)
-
- if not args.target_project and args.recreate:
- target.import_project()
-
- phab = Phab(args, target)
-
- if phab.tasks:
- target.import_from_phab(phab, args.start_at)
-
-
-if __name__ == '__main__':
- main()
diff --git a/fdo-import-repo.py b/fdo-import-repo.py
deleted file mode 100755
index 4976cea127d23d325674171c96e5c1791eb8d874..0000000000000000000000000000000000000000
--- a/fdo-import-repo.py
+++ /dev/null
@@ -1,221 +0,0 @@
-#!/usr/bin/env python3
-
-# freedesktop.org GitLab import script
-#
-# This script will read from stdin line-by-line, with the following format:
-# fdorepo/name gitlabgroup/gitlabproject
-#
-# It will:
-# - disable direct pushes to ssh://git.freedesktop.org/git/fdorepo/name
-# and enable force-pushes so it can be a perfect mirror
-# - create a GitLab repo at https://gitlab.freedesktop.org/gitlabgroup/gitlabproject
-# - mirror the content from the fd.o repo into the GitLab repo
-#
-# It does _not_ set up mirroring from GitLab to fd.o, as this requires direct
-# shell access to the GitLab cluster. To do so, please send admins entries in
-# the same format you feed as input to this script.
-#
-# This requires you have sufficient shell access to kemper.freedesktop.org, and
-# that the 'gitlab-mirror' user has been added to the correct group on LDAP.
-#
-# Author: Daniel Stone <daniels@collabora.com>
-
-import argparse
-import json
-import os
-import subprocess
-import sys
-import time
-import traceback
-import urllib.parse
-
-import gitlab
-
-class Config:
- def __init__(self):
- self.gitlab_token = None
- self.gitlab = None
- self.kemper_host = None
- self.issues = False
- self.merge_requests = False
- self.gitlab = None
-
-class Repo:
- def __init__(self, config, url_fdo, url_gitlab):
- self.config = config
- self.url_gitlab = url_gitlab
- self.url_fdo = url_fdo
- self.project = None
- self.imported = False
-
- def gitlab_repo_file_path(self):
- return "/gitlab-data/git-data/repositories/%s.git" % self.url_gitlab.lower()
-
- def gitlab_repo_mirror_cmd(self):
- return 'pushd %s && git config --local fdo.mirror-dir %s && mkdir %s/custom_hooks && ln -s /gitlab-ssh-keys/git-post-receive-mirror %s/custom_hooks/post-receive; popd' % (self.gitlab_repo_file_path(), self.url_fdo, self.gitlab_repo_file_path(), self.gitlab_repo_file_path())
-
- def legacy_file_path(self):
- return "/srv/git.freedesktop.org/git/%s.git" % self.url_fdo
-
- def legacy_clone_path(self):
- return "git://anongit.freedesktop.org/git/%s" % self.url_fdo
-
- def prepare_repo_kemper(self):
- # Accept non-fast-forwards (to make a perfect mirror), and disable
- # direct user pushes, as the only pushes will come from GitLab.
- cmd = ["ssh", self.config.kemper_host, "sh", "-c",
- "'GIT_DIR=%s git config --local receive.denynonfastforwards false && ln -s /srv/git.freedesktop.org/hooks/pre-receive-gitlab %s/hooks/pre-receive'" % (self.legacy_file_path(), self.legacy_file_path())]
- subprocess.run(cmd, check=True)
-
- def rollback_repo_kemper(self):
- # Accept non-fast-forwards (to make a perfect mirror), and disable
- # direct user pushes, as the only pushes will come from GitLab.
- cmd = ["ssh", self.config.kemper_host, "sh", "-c",
- "'GIT_DIR=%s git config --local receive.denynonfastforwards true && rm -f %s/hooks/pre-receive'" % (self.legacy_file_path(), self.legacy_file_path())]
- subprocess.run(cmd, check=True)
-
- def get_namespace_id(self):
- # Surely there has to be a cleaner way to do this ... ?
- namespace = self.url_gitlab.split('/')[:-1]
- for ns in self.config.gitlab.namespaces.list(search=namespace[-1]):
- if ns.full_path.lower() == "/".join(namespace).lower():
- return ns.id
- raise Exception("Couldn't find GitLab namespace %s" % namespace)
-
- def begin_gitlab_import(self):
- self.project = self.config.gitlab.projects.create({
- "name": self.url_gitlab.split('/')[-1],
- "namespace_id": self.get_namespace_id(),
- "import_url": self.legacy_clone_path(),
- "issues_enabled": self.config.issues,
- "merge_requests_enabled": self.config.merge_requests,
- "wiki_enabled": False,
- "merge_method": "ff",
- "visibility": "public"
- })
-
- def get_import_status(self):
- self.config.gitlab.session.headers = { "PRIVATE-TOKEN": self.config.gitlab_token }
- url = "https://gitlab.freedesktop.org/api/v4/projects/%d" % self.project.id
- ret = self.config.gitlab.session.get(url)
- if ret.status_code != 200:
- raise Exception("Status query for %s failed: %s" % (self.url_gitlab, ret.text))
- return json.loads(ret.text).get("import_status") == "finished"
-
-
-
-def main():
- parser = argparse.ArgumentParser(description="fd.o GitLab repo import",
- epilog="""
-This script imports a repository from fd.o's old Git hosting into GitLab.
-
-It accepts input on stdin in the following format, one per line:
-fdorepo/name gitlabgroup/gitlabproject
-
-For example:
-wayland/wayland wayland/wayland
-wayland/weston wayland/weston
-wayland/wayland-web wayland/wayland.freedesktop.org
-^D
-
-You will need SSH access to kemper (either being in the group, or being root),
-as well as a GitLab access token (user menu -> settings -> access tokens) for
-an admin account. To make this quicker, you almost certainly want a master
-connection active.
-
-The script will create the GitLab project itself, as well as disabling pushes
-to the old repository. It will _not_ set up mirroring from GitLab to the old
-repository, which will need to be done by someone with shell access to the
-Kubernetes cluster.
-
-Please co-ordinate with an admin when doing this script, and later send them
-a copy of the repository list you imported.
-""")
-
- parser.add_argument("--kemper-host",
- default="kemper.freedesktop.org",
- help="Hostname to pass to SSH for access to kemper",
- required=True)
- parser.add_argument("--gitlab-token",
- help="GitLab access token",
- required=True)
- parser.add_argument("--issues",
- default=False,
- action="store_true",
- help="Enable issues on migrated repos")
- parser.add_argument("--merge-requests",
- default=False,
- action="store_true",
- help="Enable merge requests on migrated repos")
- config = Config()
- parser.parse_args(namespace=config)
- config.gitlab = gitlab.Gitlab("https://gitlab.freedesktop.org",
- config.gitlab_token,
- api_version=4)
-
- repos = []
-
- for line in sys.stdin:
- try:
- (url_fdo, url_gitlab) = line[:-1].split(' ')
- except:
- print("Malformed line '%s': must be in format fdorepo/name gitlabgroup/gitlabproject" % line[:-1])
- continue
-
- repo = Repo(config, url_fdo, url_gitlab)
- repos.append(repo)
-
- try:
- repo.prepare_repo_kemper()
- except Exception as e:
- print("Failed to migrate %s to %s: '%s'" % (url_fdo, url_gitlab, e))
- traceback.print_exc()
- continue
-
- try:
- repo.begin_gitlab_import()
- except Exception as e:
- print("Failed to migrate %s to %s: '%s'" % (url_fdo, url_gitlab, e))
- traceback.print_exc()
- continue
-
- all_imported = False
- while not all_imported:
- time.sleep(2)
- all_imported = True
- for repo in repos:
- if repo.project and not repo.imported:
- repo.imported = repo.get_import_status()
- if not repo.imported:
- all_imported = False
-
- print("SUCCESSFULLY MIGRATED:")
- print("")
- print("")
- for repo in repos:
- if repo.project:
- print("%s -> %s" % (repo.url_fdo, repo.url_gitlab))
-
- print("")
- print("")
- print("")
- print("FAILED MIGRATION:")
- for repo in repos:
- if not repo.project:
- print("%s -> %s" % (repo.url_fdo, repo.url_gitlab))
- repo.rollback_repo_kemper()
-
- print("")
- print("")
- print("")
- print("Run on GitLab Kubernetes cluster:")
- for repo in repos:
- print(repo.gitlab_repo_mirror_cmd())
-
- print("")
- print("")
- print("")
- print("")
- print("Done!")
-
-main()
diff --git a/setup.py b/setup.py
index 752bb49d466c34af00f424c3a479c561e3ac4222..3620f3d84caebfdf9125eeec554e6c04870f94fc 100644
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,7 @@ setup(
'phabtogl=bztogl.phabtogl:main'],
},
- install_requires=['python-bugzilla', 'python-gitlab', 'phabricator'],
+ install_requires=['python-bugzilla', 'python-gitlab'],
setup_requires=['pytest-runner'],
tests_require=['pytest'],