From d91dc46a87466fc404a19a807a4f41d25b0baada Mon Sep 17 00:00:00 2001 From: Vincent A Date: Sat, 2 Nov 2013 01:20:45 +0100 Subject: [PATCH] github: let backend drive the requests --- modules/github/backend.py | 84 +++++++++++++++++++++++++++++-- modules/github/browser.py | 102 ++++++++++++++------------------------ 2 files changed, 115 insertions(+), 71 deletions(-) diff --git a/modules/github/backend.py b/modules/github/backend.py index 62f9d564..140f65b2 100644 --- a/modules/github/backend.py +++ b/modules/github/backend.py @@ -20,7 +20,7 @@ from weboob.tools.backend import BaseBackend, BackendConfig from weboob.tools.value import Value, ValueBackendPassword -from weboob.capabilities.bugtracker import ICapBugTracker, Issue +from weboob.capabilities.bugtracker import ICapBugTracker, Issue, Project, User, Version, Status, Update, Attachment from .browser import GithubBrowser @@ -28,6 +28,10 @@ from .browser import GithubBrowser __all__ = ['GithubBackend'] +STATUSES = {'open': Status('open', u'Open', Status.VALUE_NEW), + 'closed': Status('closed', u'closed', Status.VALUE_RESOLVED)} +# TODO tentatively parse github "labels"? + class GithubBackend(BaseBackend, ICapBugTracker): NAME = 'github' DESCRIPTION = u'GitHub issues tracking' @@ -49,10 +53,20 @@ class GithubBackend(BaseBackend, ICapBugTracker): return self.create_browser(username, password) def get_project(self, _id): - return self.browser.get_project(_id) + d = self.browser.get_project(_id) + + project = Project(_id, d['name']) + project.members = list(self._iter_members(project.id)) + project.statuses = list(STATUSES.values()) + project.categories = [] + project.versions = list(self._iter_versions(project.id)) + + return project def get_issue(self, _id): - return self.browser.get_issue(_id) + project_id, issue_number = _id.rsplit('/', 1) + project = self.get_project(project_id) + return self._make_issue(self.browser.get_issue(project_id, issue_number), project) def iter_issues(self, query): if ((query.assignee, query.author, query.status, query.title) == @@ -61,12 +75,14 @@ class GithubBackend(BaseBackend, ICapBugTracker): else: it = self.browser.iter_issues(query) - for issue in it: + project = self.get_project(query.project) + for d in it: + issue = self._make_issue(d, project) yield issue def create_issue(self, project_id): issue = Issue(0) - issue.project = self.browser.get_project(project_id) + issue.project = self.get_project(project_id) return issue def post_issue(self, issue): @@ -79,3 +95,61 @@ class GithubBackend(BaseBackend, ICapBugTracker): # iter_projects, remove_issue are impossible + def _iter_members(self, project_id): + for d in self.browser.iter_members(project_id): + yield User(d['id'], d['name']) + + def _iter_versions(self, project_id): + for d in self.browser.iter_milestones(project_id): + yield Version(d['id'], d['name']) + + def _make_issue(self, d, project): + _id = '%s/%s' % (project.id, d['number']) + issue = Issue(_id) + issue.project = project + issue.title = d['title'] + issue.body = d['body'] + issue.creation = d['creation'] + issue.updated = d['updated'] + issue.author = project.find_user(d['author'], None) + if not issue.author: + # may duplicate users + issue.author = User(d['author'], d['author']) + issue.status = STATUSES[d['status']] + + if d['assignee']: + issue.assignee = project.find_user(d['assignee'], None) + else: + issue.assignee = None + + if d['version']: + issue.version = project.find_version(d['version'], None) + else: + issue.version = None + + issue.category = None + + issue.attachments = [self._make_attachment(dattach) for dattach in d['attachments']] + + issue.history = [] + issue.history += [self._make_comment(dcomment, project) for dcomment in d['comments']] + + return issue + + def _make_attachment(self, d): + a = Attachment(d['url']) + a.url = d['url'] + a.filename = d['filename'] + return a + + def _make_comment(self, d, project): + u = Update(d['id']) + u.message = d['message'] + u.author = project.find_user(d['author'], None) + if not u.author: + # may duplicate users + u.author = User(d['author'], d['author']) + u.date = d['date'] + u.changes = [] + u.attachments = [self._make_attachment(dattach) for dattach in d['attachments']] + return u diff --git a/modules/github/browser.py b/modules/github/browser.py index 1aaf2ebd..5cd5dd58 100644 --- a/modules/github/browser.py +++ b/modules/github/browser.py @@ -19,7 +19,6 @@ from weboob.tools.browser import BaseBrowser -from weboob.capabilities.bugtracker import Issue, Project, User, Version, Status, Update, Attachment from weboob.tools.json import json as json_module from base64 import b64encode import datetime @@ -31,9 +30,6 @@ from urllib import quote_plus __all__ = ['GithubBrowser'] -STATUSES = {'open': Status('open', u'Open', Status.VALUE_NEW), - 'closed': Status('closed', u'closed', Status.VALUE_RESOLVED)} -# TODO tentatively parse github "labels"? class GithubBrowser(BaseBrowser): PROTOCOL = 'https' @@ -48,27 +44,20 @@ class GithubBrowser(BaseBrowser): def home(self): pass - def get_project(self, _id): - json = self.do_get('https://api.github.com/repos/%s' % _id) + def get_project(self, project_id): + json = self.do_get('https://api.github.com/repos/%s' % project_id) + return {'name': json['name'], 'id': project_id} - project = Project(_id, json['name']) - project.members = list(self.iter_members(_id)) - project.statuses = list(STATUSES.values()) - project.categories = [] - project.versions = list(self._get_milestones(_id)) - return project - - def get_issue(self, _id, fetch_project=True): - project_id, issue_number = _id.rsplit('/', 1) + def get_issue(self, project_id, issue_number): json = self.do_get('https://api.github.com/repos/%s/issues/%s' % (project_id, issue_number)) - return self.make_issue(_id, json, fetch_project) + return self._make_issue(project_id, issue_number, json) def iter_project_issues(self, project_id): base_url = 'https://api.github.com/repos/%s/issues' % project_id for json in self._paginated(base_url): for jissue in json: - issue_id = '%s/%s' % (project_id, jissue['number']) - yield self.make_issue(issue_id, jissue) + issue_number = jissue['number'] + yield self._make_issue(project_id, issue_number, jissue) if len(json) < 100: break @@ -82,14 +71,14 @@ class GithubBrowser(BaseBrowser): qsparts.append('state:%s' % query.status) if query.title: qsparts.append('%s in:title' % query.title) - + qs = quote_plus(' '.join(qsparts)) base_url = 'https://api.github.com/search/issues?q=%s' % qs for json in self._paginated(base_url): for jissue in json['items']: - issue_id = '%s/%s' % (query.project, jissue['number']) - yield self.make_issue(issue_id, jissue) + issue_number = jissue['number'] + yield self._make_issue(query.project, issue_number, jissue) if not len(json['items']): break @@ -102,8 +91,8 @@ class GithubBrowser(BaseBrowser): base_data = json_module.dumps(data) url = 'https://api.github.com/repos/%s/issues' % issue.project.id json = self.do_post(url, base_data) - issue_id = '%s/%s' % (issue.project.id, json['id']) - return self.make_issue(issue_id, json) + issue_number = json['id'] + return self._make_issue(issue.project.id, issue_number, json) def post_comment(self, issue_id, comment): project_id, issue_number = issue_id.rsplit('/', 1) @@ -112,56 +101,41 @@ class GithubBrowser(BaseBrowser): self.do_post(url, data) # helpers - def make_issue(self, _id, json, fetch_project=True): - project_id, issue_number = _id.rsplit('/', 1) - issue = Issue(_id) - issue.title = json['title'] - issue.body = json['body'] - issue.category = None - issue.creation = parse_date(json['created_at']) - issue.updated = parse_date(json['updated_at']) - issue.attachments = list(self._get_attachments(issue.body)) - if fetch_project: - issue.project = self.get_project(project_id) - issue.author = self.get_user(json['user']['login']) + def _make_issue(self, project_id, issue_number, json): + d = {'number': issue_number, 'title': json['title'], 'body': json['body'], 'creation': parse_date(json['created_at']), 'updated': parse_date(json['updated_at']), 'author': json['user']['login'], 'status': json['state']} + if json['assignee']: - issue.assignee = self.get_user(json['assignee']['login']) + d['assignee'] = json['assignee']['login'] else: - issue.assignee = None - issue.status = STATUSES[json['state']] + d['assignee'] = None if json['milestone']: - issue.version = self.make_milestone(json['milestone']) - if json['comments'] > 0: - issue.history = [comment for comment in self.get_comments(project_id, issue_number)] + d['version'] = json['milestone'] else: - issue.history = [] + d['version'] = None + if json['comments'] > 0: + d['comments'] = list(self.get_comments(project_id, issue_number)) + else: + d['comments'] = [] + d['attachments'] = list(self._extract_attachments(d['body'])) + # TODO fetch other updates? - return issue + return d - def _get_milestones(self, project_id): + def iter_milestones(self, project_id): for jmilestone in self.do_get('https://api.github.com/repos/%s/milestones' % project_id): - yield self.make_milestone(jmilestone) - - def make_milestone(self, json): - return Version(json['number'], json['title']) + yield {'id': jmilestone['number'], 'name': jmilestone['title']} def get_comments(self, project_id, issue_number): json = self.do_get('https://api.github.com/repos/%s/issues/%s/comments' % (project_id, issue_number)) for jcomment in json: - comment = Update(jcomment['id']) - comment.message = jcomment['body'] - comment.author = self.make_user(jcomment['user']['login']) - comment.date = parse_date(jcomment['created_at']) - comment.changes = [] - comment.attachments = list(self._get_attachments(comment.message)) - yield comment + d = {'id': jcomment['id'], 'message': jcomment['body'], 'author': jcomment['user']['login'], 'date': parse_date(jcomment['created_at'])} + d['attachments'] = list(self._extract_attachments(d['message'])) + yield d - def _get_attachments(self, message): + def _extract_attachments(self, message): for attach_url in re.findall(r'https://f.cloud.github.com/assets/[\w/.-]+', message): - attach = Attachment(attach_url) - attach.url = attach_url - attach.filename = os.path.basename(attach_url) - yield attach + d = {'url': attach_url, 'filename': os.path.basename(attach_url)} + yield d def _paginated(self, url, start_at=1): while True: @@ -178,16 +152,12 @@ class GithubBrowser(BaseBrowser): name = json['name'] else: name = _id # wasted one request... - return User(_id, name) - - def make_user(self, name): - return User(name, name) + return {'id': _id, 'name': name} def iter_members(self, project_id): for json in self._paginated('https://api.github.com/repos/%s/assignees' % project_id): for jmember in json: - user = self.make_user(jmember['login']) # no request, no name - yield user + yield {'id': jmember['login'], 'name': jmember['login']} if len(json) < 100: break