From 68919a77d3bbec993284b431ea02a56b7d77c06f Mon Sep 17 00:00:00 2001 From: Vincent A Date: Sun, 22 Sep 2013 12:38:30 +0200 Subject: [PATCH] new bugtracker backend: github --- modules/github/__init__.py | 24 +++++ modules/github/backend.py | 75 +++++++++++++ modules/github/browser.py | 210 +++++++++++++++++++++++++++++++++++++ modules/github/favicon.png | Bin 0 -> 4537 bytes modules/github/test.py | 49 +++++++++ 5 files changed, 358 insertions(+) create mode 100644 modules/github/__init__.py create mode 100644 modules/github/backend.py create mode 100644 modules/github/browser.py create mode 100644 modules/github/favicon.png create mode 100644 modules/github/test.py diff --git a/modules/github/__init__.py b/modules/github/__init__.py new file mode 100644 index 00000000..21b1de23 --- /dev/null +++ b/modules/github/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from .backend import GithubBackend + + +__all__ = ['GithubBackend'] diff --git a/modules/github/backend.py b/modules/github/backend.py new file mode 100644 index 00000000..237656ae --- /dev/null +++ b/modules/github/backend.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.backend import BaseBackend, BackendConfig +from weboob.tools.value import Value, ValueBackendPassword +from weboob.capabilities.bugtracker import ICapBugTracker, Issue + +from .browser import GithubBrowser + + +__all__ = ['GithubBackend'] + + +class GithubBackend(BaseBackend, ICapBugTracker): + NAME = 'github' + DESCRIPTION = u'GitHub issues tracking' + MAINTAINER = u'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'AGPLv3+' + VERSION = '0.h' + CONFIG = BackendConfig(Value('username', label='Username', default=''), + ValueBackendPassword('password', label='Password', default='')) + + BROWSER = GithubBrowser + + def create_default_browser(self): + username = self.config['username'].get() + if username: + password = self.config['password'].get() + else: + password = None + return self.create_browser(username, password) + + def get_project(self, _id): + return self.browser.get_project(_id) + + def get_issue(self, _id): + return self.browser.get_issue(_id) + + def iter_issues(self, query): + for issue in self.browser.iter_issues(query): + yield issue + + def create_issue(self, project_id): + issue = Issue(0) + issue.project = self.browser.get_project(project_id) + return issue + + def post_issue(self, issue): + assert not issue.attachments + self.browser.post_issue(issue) + + def update_issue(self, issue_id, update): + assert not update.attachments + self.browser.post_comment(issue_id, update.message) + + # iter_projects, remove_issue are impossible + diff --git a/modules/github/browser.py b/modules/github/browser.py new file mode 100644 index 00000000..79bbe6b0 --- /dev/null +++ b/modules/github/browser.py @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.browser import BaseBrowser +from weboob.capabilities.bugtracker import Issue, Project, User, Version, Status, Update, Attachment +from weboob.tools.json import json as json_module +from base64 import b64encode +import datetime +import re +import os +from urllib import quote_plus + + +__all__ = ['GithubBrowser'] + + +STATUSES = {'open': Status('open', u'Open', Status.VALUE_NEW), + 'closed': Status('closed', u'closed', Status.VALUE_RESOLVED)} +# TODO tentatively parse github "labels"? + +class GithubBrowser(BaseBrowser): + PROTOCOL = 'https' + DOMAIN = 'api.github.com' + ENCODING = 'utf-8' + + def __init__(self, *a, **kw): + kw['parser'] = 'json' + BaseBrowser.__init__(self, *a, **kw) + self.fewer_requests = not bool(self.username) + + def home(self): + pass + + def get_project(self, _id): + json = self.do_get('https://api.github.com/repos/%s' % _id) + + project = Project(_id, json['name']) + project.members = list(self.iter_members(_id)) + project.statuses = list(STATUSES.values()) + project.categories = [] + project.versions = list(self._get_milestones(_id)) + return project + + def get_issue(self, _id, fetch_project=True): + project_id, issue_number = _id.rsplit('/', 1) + json = self.do_get('https://api.github.com/repos/%s/issues/%s' % (project_id, issue_number)) + return self.make_issue(_id, json, fetch_project) + + def iter_issues(self, query): + qsparts = ['repo:%s' % query.project] + if query.assignee: + qsparts.append('assignee:%s' % query.assignee) + if query.author: + qsparts.append('author:%s' % query.author) + if query.status: + qsparts.append('state:%s' % query.status) + if query.title: + qsparts.append('%s in:title' % query.title) + + qs = quote_plus(' '.join(qsparts)) + + base_url = 'https://api.github.com/search/issues?q=%s' % qs + for json in self._paginated(base_url): + for jissue in json['items']: + issue_id = '%s/%s' % (query.project, jissue['number']) + yield self.make_issue(issue_id, jissue) + if not len(json['items']): + break + + def post_issue(self, issue): + data = {'title': issue.title, 'body': issue.body} + if issue.assignee: + data['assignee'] = issue.assignee.id + if issue.version: + data['milestone'] = issue.version.id + base_data = json_module.dumps(data) + url = 'https://api.github.com/repos/%s/issues' % issue.project.id + json = self.do_post(url, base_data) + issue_id = '%s/%s' % (issue.project.id, json['id']) + return self.make_issue(issue_id, json) + + def post_comment(self, issue_id, comment): + project_id, issue_number = issue_id.rsplit('/', 1) + url = 'https://api.github.com/repos/%s/issues/%s/comments' % (project_id, issue_number) + data = json_module.dumps({'body': comment}) + self.do_post(url, data) + + # helpers + def make_issue(self, _id, json, fetch_project=True): + project_id, issue_number = _id.rsplit('/', 1) + issue = Issue(_id) + issue.title = json['title'] + issue.body = json['body'] + issue.category = None + issue.creation = parse_date(json['created_at']) + issue.updated = parse_date(json['updated_at']) + issue.attachments = list(self._get_attachments(issue.body)) + if fetch_project: + issue.project = self.get_project(project_id) + issue.author = self.get_user(json['user']['login']) + if json['assignee']: + issue.assignee = self.get_user(json['assignee']['login']) + else: + issue.assignee = None + issue.status = STATUSES[json['state']] + if json['milestone']: + issue.version = self.make_milestone(json['milestone']) + if json['comments'] > 0: + issue.history = [comment for comment in self.get_comments(project_id, issue_number)] + else: + issue.history = [] + # TODO fetch other updates? + return issue + + def _get_milestones(self, project_id): + for jmilestone in self.do_get('https://api.github.com/repos/%s/milestones' % project_id): + yield self.make_milestone(jmilestone) + + def make_milestone(self, json): + return Version(json['number'], json['title']) + + def get_comments(self, project_id, issue_number): + json = self.do_get('https://api.github.com/repos/%s/issues/%s/comments' % (project_id, issue_number)) + for jcomment in json: + comment = Update(jcomment['id']) + comment.message = jcomment['body'] + comment.author = self.make_user(jcomment['user']['login']) + comment.date = parse_date(jcomment['created_at']) + comment.changes = [] + comment.attachments = list(self._get_attachments(comment.message)) + yield comment + + def _get_attachments(self, message): + for attach_url in re.findall(r'https://f.cloud.github.com/assets/[\w/.-]+', message): + attach = Attachment(attach_url) + attach.url = attach_url + attach.filename = os.path.basename(attach_url) + yield attach + + def _paginated(self, url, start_at=1): + while True: + if '?' in url: + page_url = '%s&per_page=100&page=%s' % (url, start_at) + else: + page_url = '%s?per_page=100&page=%s' % (url, start_at) + yield self.do_get(page_url) + start_at += 1 + + def get_user(self, _id): + json = self.do_get('https://api.github.com/users/%s' % _id) + if 'name' in json: + name = json['name'] + else: + name = _id # wasted one request... + return User(_id, name) + + def make_user(self, name): + return User(name, name) + + def iter_members(self, project_id): + for json in self._paginated('https://api.github.com/repos/%s/assignees' % project_id): + for jmember in json: + user = self.make_user(jmember['login']) # no request, no name + yield user + if len(json) < 100: + break + + def do_get(self, url): + headers = self.auth_headers() + headers.update({'Accept': 'application/vnd.github.preview'}) + req = self.request_class(url, None, headers=headers) + return self.get_document(self.openurl(req)) + + def do_post(self, url, data): + headers = self.auth_headers() + headers.update({'Accept': 'application/vnd.github.preview'}) + req = self.request_class(url, data, headers=headers) + return self.get_document(self.openurl(req)) + + def auth_headers(self): + if self.username: + return {'Authorization': 'Basic %s' % b64encode('%s:%s' % (self.username, self.password))} + else: + return {} + +# TODO use a cache for objects and/or pages? +# TODO use an api-key? + +def parse_date(s): + if s.endswith('Z'): + s = s[:-1] + + return datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%S') diff --git a/modules/github/favicon.png b/modules/github/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..2bf665084755a1bb5669689ba3ccb685ff653fdf GIT binary patch literal 4537 zcmWkyc|6m97+*qjUArQz(B*OR#cpDtuP-gI?df%A@ zFO~qr^&kj@jrZt21|jDQfyEQSNK2Cw6Rcu_ij0MIW`STyIM~EK*vS9(ZFj$5h*6-s zYp^>;GSoZROVSi+X>%=>Ll6StRz(_LvJ3mNdczmXj~=-<7x8lSwSj*f>FN!K8?k}h zYEmeenUo{n0~g%QUx|Y6;&X&jlRxS2ALli?#Z`3X*vamAGrl7HRRc+zEB`$vk*>&> zHqe8<1nH}Kdh8r@`1fC&9~N-_6|H|)E~~`TV|^^L!n0z#yxvnao(tD{$8A&7Ka1y` zi!dyzL_*!t2*%SkFu;|Nn8=-MtbBLLF8k4=)0&!s@ZP`mOkByv0dpUOqYk%68iN;k z{@l|qu`W!r%+qk)UK$F}ejIcA*zYJkQSD%8YHBJJZ#bZKvcYcxH?oW$!*&loGykW8 zc;ATgQX^apU6T|@S=_1g7SYz$-k6MBcb)(Exbg0?fF1fUruzM6l>Jo+$zBzx6!I|| zZ4w%)8L~1OTU~t~JO1jVjEu~`T7(Unzo@8aBTQ#Xi^Igk1nfSguOD4-l|-cSNC5Ihy{HhujbdF*S9% zSE)5#o-q3A_9<32wl?JtF_w^IV+IgRN20{NBsW^xrOaZO9g#?s%s^PI99L0KaDAt2 zL3V*M$9Z{q8CY3yX!uVOaX6g57z???6^mt`?=PgSO*GL3I3e&}N?jeF;-lqBMhHYU zi=_r`M6ZKMv2${|fgPZU9btR3k5E^y?vz4fxo}}RX&*mY9c3Q`+Bx*%npAJ#ms*Pa zt!|=ll|#j@A_R?oO7#)>z#=6iBGT?z7@Yu8wL0EV)ZF}sGnBX)CR$NZG5R)A^WNU9 zATce?kX}dZCs1-Snl?fe?wl@G$?a%~KCrQ|xqo~7)uAi#dHDF~TN z=Hf`CrS7tKV2ow-ofDIxOxrBo0vw5(hr8e0hN$T0FJ3gwn|MUDmUt)8W*VdRRCd-T zQsPcw`9TBO-|UALofK0KO3l-B0gaMUEA^05XGoTNc! zad}FqLn#!^CsS0mk*n=cag|R%pdF*)Zj3uGjj2(31EEF}L`m*h`JS?chVuqA*?0qz zdwhp0sO@reSiy?H9u*?T#Vhi-mbGyy~Jg*_ZSo8x>}%vMQ#_z2|p|7QF>ofZv7cSUVB#O z;dsAVA8ma0Os~LekdBV&`(xOTiuY~gC4|xD%xmM|HVrqKA4vhz8lVZ$3{AlskS5JR$TqMUMt*)>% zVOxK}`5TDfv}xLtrlwN_3hboWo2I57h010_QH~SS;RQJsvJ5xBrnm~V*T7wcWM|%P z?$^N0!wRgI{YB|vpCV^c&DU$;=EkTGT7J&X;$v^_DlD$7z^GInL+CWWN!#n7`GLVf zT!2u-oF6i`c zJxl<^T%4MBE@*o%N-JbVXlZ#lM$q8eH9?0PH@0~OF$5t|nv}l2gthKA-_>&aSdfwS z)sQ3T<0LUrn?zyNOs<=s1$t|=cQttw+?dZo89e&POdvA}t{V$Q>6Dfhas9u)Plv32 z!TXN9lv#Dfm?f6+uB@*o_Rk-eVvKTekpQX!mC}aOf}#L*&3(uX18P$pIBVAEsOaQ1 zCW&zQSOeEeGp96l>+$W_<$WGw!9gtPnVCwLxNIC9iGnUs+;}|x_m3>=fbAvpTa|nQ zgJDvd-*wVIW(k6m*w_lOX1gv@ZODs!-ciAg#LLWOUsi40s-iOJqh3E(eZP(7iDjs zL?Q{~UkGBBv&tVRGLij`F=p0RR=yvmqbC(qJBGDy4m&>?J}>qmS5+?giVr!#SUMxS zdywb4tu3#Jh)BRp_em~8<;Wc@cC_l4wcZ{Kt8v2ts~H$ca_45nSH=9hw=&Tr2#VXU zyDRD(xvHN1@ZlnZYF?jx)y-p$ih^kp`e^{2^DMbJXF8N4G81IXNn8c+6fF zt21CcEVr=a5^gj)xy0L;URNCH265{DQTF!p8~#>CJ&WSQzpt^-5$6nFu4q62I988gg^iv`LW^r^ zGNbkvOFQScxOe`0sDLE*V`w2wVkrXW+Hp*5nS)4HIX5#bw%VS3VqyZ>LjY-wcx%Ha zQ;}W`ktRq(o>WwveEasTc2u-p;aV+GwpW=w-%3uy?`u-UjRp#pdNx(Vwr*#8dwOHK zV_|vuc%g$)X^40H_d0KSRq*1KAM^92&)R0Hm9l!WNzv!ok3Z^VO6gLOI@~@u)Qa9$ zqkMY)HxGu4AX`0_Ro74H!YwWN@bUAvO$XehrHM60AB1FDQ( zaWUrfc{rU84-c1DP@v4EZo|RfZJ0pDl&;J> z{_n6F%XpR|VWAG#x$5-PRQKr0YBLy=fhRd$oMFOG6LdYDRq}FiEu4+iZO_Un5}Dc> zt|MMoU%yzLB;Vy_Fsmps{J~u(-;`Cy(dfV)97f9+7yO*KG)2bB5(Bu64(%~kB^ehFi;ly%>DQc zic(AO%hPllt8oVeB^(;DU>L0@v9mH3H#av|-0w*&$wUg3i8SXip*RZ(XSGMLAr;;t z2?+_x8XC-(xPG10kkPL*AZKMAfA#Vu2jE7(jRFL7nZiWqNRy`0}OVpZ(>^jlC%CK27bp?d1`-)i3p}>zY^W z(ROxIW1fXwfB>WjlzS+e>sN+W{rh*~(}(1#_wS)Q>)+A=iFe+rE>4#bN@z3U)QtzOZ~2?ro%Pm# ze-DDc`!$?mVq)6IF*7p;MXZ5=TaGj#KgLQsm23)4IFLpkl;$lMdye`nR3V1 zJOU}R-_$RCu|vzW-F)EK5|z6$S{-M1^(xOLt|Uol{7hqAaq@h)O6Z#7;OHm<^kBgU zAtf{#2^Tnh{P^+W*49Gcekk&0UxISURO*lY?d2bFoO+Tef=aUf&6~^@B7VsW3kyqS z!njRPNmME<1HqL}Gt%2%IQ?@fj+2Mv&O*C`R5#7)F&?gTu?ADcuL#LK{nQe56V!X) zaeIKJ?ji6iAi~2ha5KhM!C$zD(zp{`q5WIMuU?U&{_f3pE^jYo{Rd-rd-VtbL+FUh z0qX0Vxx~ei7W92?#s>14Lqj(vc1M zu_x}-MSAHn#}G#4X=9_%)_lLIlhb2o?b6pc7*Z?dug=HE7ZYvdi(6GiQg4BO03ZbZ zNyjJ?1}qCB!!b|_c|xXLz&UyOcO{m2RelrN0AB4LweBaIK1}2bQyd!a%rGAX`}Hf) z{E;F!h2(A8WiCcwdyN3dwM04i1O+=*Lpo*yrIeI7tx9=;k_sw18u4277@aGx$ssShxR&dk+Db~+FiSm1rpUL?20o=w82I`5 zbM#~MK#7CxK>aNBrr$`==AL`Ab8s9rwP$}pSG5~0;9%!ASdj+vHN%ta2+G@NhaC?g zW+1@s?(S7%*oe-u7cV->uh-f*I1t*~4JSUXYeZt!c3)pIK~3!*d;%la@<`>(%2;jV zy+du_ZsZA|^kCKoPbzgO@?Q6bEJ6Bc@CUK|n43HDXgL={6$68UjzMfoEMmu36Z$;p z9TU1Sl^dpF`NgGHmrYH#y)G?p2De(H_&_Ot?e3lyNnIKvcT5|YnBWy54dWI5#hA1G zho%zPKf-dRv;&gT5Z!j(OCv9g*krURx~npV+lD)`0% NL7G?@zcjpw`yV3DfRz9M literal 0 HcmV?d00001 diff --git a/modules/github/test.py b/modules/github/test.py new file mode 100644 index 00000000..bcb8f73b --- /dev/null +++ b/modules/github/test.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.test import BackendTest +from weboob.capabilities.bugtracker import Query + + +class GithubTest(BackendTest): + BACKEND = 'github' + + def test_project(self): + project = self.backend.get_project('github/hubot') + assert project + assert project.name + assert project.members + + def test_issue(self): + issue = self.backend.get_issue('github/hubot/1') + assert issue + assert issue.title + assert issue.body + assert issue.creation + assert issue.history + + def test_search(self): + query = Query() + query.project = u'github/hubot' + query.status = u'closed' + query.title = u'fix' + issues = self.backend.iter_issues(query) + issue = issues.next() + assert issue.status.name == 'closed'