github: let backend drive the requests

modules/github/backend.py

@@ -20,7 +20,7 @@
 
 from weboob.tools.backend import BaseBackend, BackendConfig
 from weboob.tools.value import Value, ValueBackendPassword
-from weboob.capabilities.bugtracker import ICapBugTracker, Issue
+from weboob.capabilities.bugtracker import ICapBugTracker, Issue, Project, User, Version, Status, Update, Attachment
 
 from .browser import GithubBrowser
 
@@ -28,6 +28,10 @@ from .browser import GithubBrowser
 __all__ = ['GithubBackend']
 
 
+STATUSES = {'open': Status('open', u'Open', Status.VALUE_NEW),
+            'closed': Status('closed', u'closed', Status.VALUE_RESOLVED)}
+# TODO tentatively parse github "labels"?
+
 class GithubBackend(BaseBackend, ICapBugTracker):
     NAME = 'github'
     DESCRIPTION = u'GitHub issues tracking'
@@ -49,10 +53,20 @@ class GithubBackend(BaseBackend, ICapBugTracker):
         return self.create_browser(username, password)
 
     def get_project(self, _id):
-        return self.browser.get_project(_id)
+        d = self.browser.get_project(_id)
+
+        project = Project(_id, d['name'])
+        project.members = list(self._iter_members(project.id))
+        project.statuses = list(STATUSES.values())
+        project.categories = []
+        project.versions = list(self._iter_versions(project.id))
+
+        return project
 
     def get_issue(self, _id):
-        return self.browser.get_issue(_id)
+        project_id, issue_number = _id.rsplit('/', 1)
+        project = self.get_project(project_id)
+        return self._make_issue(self.browser.get_issue(project_id, issue_number), project)
 
     def iter_issues(self, query):
         if ((query.assignee, query.author, query.status, query.title) ==
@@ -61,12 +75,14 @@ class GithubBackend(BaseBackend, ICapBugTracker):
         else:
             it = self.browser.iter_issues(query)
 
-        for issue in it:
+        project = self.get_project(query.project)
+        for d in it:
+            issue = self._make_issue(d, project)
             yield issue
 
     def create_issue(self, project_id):
         issue = Issue(0)
-        issue.project = self.browser.get_project(project_id)
+        issue.project = self.get_project(project_id)
         return issue
 
     def post_issue(self, issue):
@@ -79,3 +95,61 @@ class GithubBackend(BaseBackend, ICapBugTracker):
 
     # iter_projects, remove_issue are impossible
 
+    def _iter_members(self, project_id):
+        for d in self.browser.iter_members(project_id):
+            yield User(d['id'], d['name'])
+
+    def _iter_versions(self, project_id):
+        for d in self.browser.iter_milestones(project_id):
+            yield Version(d['id'], d['name'])
+
+    def _make_issue(self, d, project):
+        _id = '%s/%s' % (project.id, d['number'])
+        issue = Issue(_id)
+        issue.project = project
+        issue.title = d['title']
+        issue.body = d['body']
+        issue.creation = d['creation']
+        issue.updated = d['updated']
+        issue.author = project.find_user(d['author'], None)
+        if not issue.author:
+            # may duplicate users
+            issue.author = User(d['author'], d['author'])
+        issue.status = STATUSES[d['status']]
+
+        if d['assignee']:
+            issue.assignee = project.find_user(d['assignee'], None)
+        else:
+            issue.assignee = None
+
+        if d['version']:
+            issue.version = project.find_version(d['version'], None)
+        else:
+            issue.version = None
+
+        issue.category = None
+
+        issue.attachments = [self._make_attachment(dattach) for dattach in d['attachments']]
+
+        issue.history = []
+        issue.history += [self._make_comment(dcomment, project) for dcomment in d['comments']]
+
+        return issue
+
+    def _make_attachment(self, d):
+        a = Attachment(d['url'])
+        a.url = d['url']
+        a.filename = d['filename']
+        return a
+
+    def _make_comment(self, d, project):
+        u = Update(d['id'])
+        u.message = d['message']
+        u.author = project.find_user(d['author'], None)
+        if not u.author:
+            # may duplicate users
+            u.author = User(d['author'], d['author'])
+        u.date = d['date']
+        u.changes = []
+        u.attachments = [self._make_attachment(dattach) for dattach in d['attachments']]
+        return u

modules/github/browser.py

@@ -19,7 +19,6 @@
 
 
 from weboob.tools.browser import BaseBrowser
-from weboob.capabilities.bugtracker import Issue, Project, User, Version, Status, Update, Attachment
 from weboob.tools.json import json as json_module
 from base64 import b64encode
 import datetime
@@ -31,9 +30,6 @@ from urllib import quote_plus
 __all__ = ['GithubBrowser']
 
 
-STATUSES = {'open': Status('open', u'Open', Status.VALUE_NEW),
-            'closed': Status('closed', u'closed', Status.VALUE_RESOLVED)}
-# TODO tentatively parse github "labels"?
 
 class GithubBrowser(BaseBrowser):
     PROTOCOL = 'https'
@@ -48,27 +44,20 @@ class GithubBrowser(BaseBrowser):
     def home(self):
         pass
 
-    def get_project(self, _id):
+    def get_project(self, project_id):
-        json = self.do_get('https://api.github.com/repos/%s' % _id)
+        json = self.do_get('https://api.github.com/repos/%s' % project_id)
+        return {'name': json['name'], 'id': project_id}
 
-        project = Project(_id, json['name'])
+    def get_issue(self, project_id, issue_number):
-        project.members = list(self.iter_members(_id))
-        project.statuses = list(STATUSES.values())
-        project.categories = []
-        project.versions = list(self._get_milestones(_id))
-        return project
-
-    def get_issue(self, _id, fetch_project=True):
-        project_id, issue_number = _id.rsplit('/', 1)
         json = self.do_get('https://api.github.com/repos/%s/issues/%s' % (project_id, issue_number))
-        return self.make_issue(_id, json, fetch_project)
+        return self._make_issue(project_id, issue_number, json)
 
     def iter_project_issues(self, project_id):
         base_url = 'https://api.github.com/repos/%s/issues' % project_id
         for json in self._paginated(base_url):
             for jissue in json:
-                issue_id = '%s/%s' % (project_id, jissue['number'])
+                issue_number = jissue['number']
-                yield self.make_issue(issue_id, jissue)
+                yield self._make_issue(project_id, issue_number, jissue)
             if len(json) < 100:
                 break
 
@@ -88,8 +77,8 @@ class GithubBrowser(BaseBrowser):
         base_url = 'https://api.github.com/search/issues?q=%s' % qs
         for json in self._paginated(base_url):
             for jissue in json['items']:
-                issue_id = '%s/%s' % (query.project, jissue['number'])
+                issue_number = jissue['number']
-                yield self.make_issue(issue_id, jissue)
+                yield self._make_issue(query.project, issue_number, jissue)
             if not len(json['items']):
                 break
 
@@ -102,8 +91,8 @@ class GithubBrowser(BaseBrowser):
         base_data = json_module.dumps(data)
         url = 'https://api.github.com/repos/%s/issues' % issue.project.id
         json = self.do_post(url, base_data)
-        issue_id = '%s/%s' % (issue.project.id, json['id'])
+        issue_number = json['id']
-        return self.make_issue(issue_id, json)
+        return self._make_issue(issue.project.id, issue_number, json)
 
     def post_comment(self, issue_id, comment):
         project_id, issue_number = issue_id.rsplit('/', 1)
@@ -112,56 +101,41 @@ class GithubBrowser(BaseBrowser):
         self.do_post(url, data)
 
     # helpers
-    def make_issue(self, _id, json, fetch_project=True):
+    def _make_issue(self, project_id, issue_number, json):
-        project_id, issue_number = _id.rsplit('/', 1)
+        d = {'number': issue_number, 'title': json['title'], 'body': json['body'], 'creation': parse_date(json['created_at']), 'updated': parse_date(json['updated_at']), 'author': json['user']['login'], 'status': json['state']}
-        issue = Issue(_id)
+
-        issue.title = json['title']
-        issue.body = json['body']
-        issue.category = None
-        issue.creation = parse_date(json['created_at'])
-        issue.updated = parse_date(json['updated_at'])
-        issue.attachments = list(self._get_attachments(issue.body))
-        if fetch_project:
-            issue.project = self.get_project(project_id)
-        issue.author = self.get_user(json['user']['login'])
         if json['assignee']:
-            issue.assignee = self.get_user(json['assignee']['login'])
+            d['assignee'] = json['assignee']['login']
         else:
-            issue.assignee = None
+            d['assignee'] = None
-        issue.status = STATUSES[json['state']]
         if json['milestone']:
-            issue.version = self.make_milestone(json['milestone'])
+            d['version'] = json['milestone']
-        if json['comments'] > 0:
-            issue.history = [comment for comment in self.get_comments(project_id, issue_number)]
         else:
-            issue.history = []
+            d['version'] = None
+        if json['comments'] > 0:
+            d['comments'] = list(self.get_comments(project_id, issue_number))
+        else:
+            d['comments'] = []
+        d['attachments'] = list(self._extract_attachments(d['body']))
+
         # TODO fetch other updates?
-        return issue
+        return d
 
-    def _get_milestones(self, project_id):
+    def iter_milestones(self, project_id):
         for jmilestone in self.do_get('https://api.github.com/repos/%s/milestones' % project_id):
-            yield self.make_milestone(jmilestone)
+            yield {'id': jmilestone['number'], 'name': jmilestone['title']}
-
-    def make_milestone(self, json):
-        return Version(json['number'], json['title'])
 
     def get_comments(self, project_id, issue_number):
         json = self.do_get('https://api.github.com/repos/%s/issues/%s/comments' % (project_id, issue_number))
         for jcomment in json:
-            comment = Update(jcomment['id'])
+            d = {'id': jcomment['id'], 'message': jcomment['body'], 'author': jcomment['user']['login'], 'date': parse_date(jcomment['created_at'])}
-            comment.message = jcomment['body']
+            d['attachments'] = list(self._extract_attachments(d['message']))
-            comment.author = self.make_user(jcomment['user']['login'])
+            yield d
-            comment.date = parse_date(jcomment['created_at'])
-            comment.changes = []
-            comment.attachments = list(self._get_attachments(comment.message))
-            yield comment
 
-    def _get_attachments(self, message):
+    def _extract_attachments(self, message):
         for attach_url in re.findall(r'https://f.cloud.github.com/assets/[\w/.-]+', message):
-            attach = Attachment(attach_url)
+            d = {'url': attach_url, 'filename': os.path.basename(attach_url)}
-            attach.url = attach_url
+            yield d
-            attach.filename = os.path.basename(attach_url)
-            yield attach
 
     def _paginated(self, url, start_at=1):
         while True:
@@ -178,16 +152,12 @@ class GithubBrowser(BaseBrowser):
             name = json['name']
         else:
             name = _id # wasted one request...
-        return User(_id, name)
+        return {'id': _id, 'name': name}
-    
-    def make_user(self, name):
-        return User(name, name)
 
     def iter_members(self, project_id):
         for json in self._paginated('https://api.github.com/repos/%s/assignees' % project_id):
             for jmember in json:
-                user = self.make_user(jmember['login']) # no request, no name
+                yield {'id': jmember['login'], 'name': jmember['login']}
-                yield user
             if len(json) < 100:
                 break