From 07a1d04bb6af18e4723fc4c35b0f4422866172da Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Sun, 30 Mar 2014 12:32:20 +0200 Subject: [PATCH] =?UTF-8?q?fix=20compatibility=20with=20redmine=202.4,=20s?= =?UTF-8?q?upport=20start/end/tracker/priority=20(courtesy=20of=20Fran?= =?UTF-8?q?=C3=A7ois=20Revol)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/redmine/backend.py | 41 ++++- modules/redmine/browser.py | 6 +- modules/redmine/pages/issues.py | 160 +++++++++++++++++- .../applications/boobtracker/boobtracker.py | 43 ++++- weboob/capabilities/bugtracker.py | 6 + 5 files changed, 234 insertions(+), 22 deletions(-) diff --git a/modules/redmine/backend.py b/modules/redmine/backend.py index c0105853..508982ac 100644 --- a/modules/redmine/backend.py +++ b/modules/redmine/backend.py @@ -120,7 +120,8 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection): raise CollectionNotFound(collection.split_path) ############# CapBugTracker ################################################### - def _build_project(self, project_dict): + @classmethod + def _build_project(cls, project_dict): project = Project(project_dict['name'], project_dict['name']) project.members = [User(int(u[0]), u[1]) for u in project_dict['members']] project.versions = [Version(int(v[0]), v[1]) for v in project_dict['versions']] @@ -129,6 +130,20 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection): project.statuses = [Status(int(s[0]), s[1], 0) for s in project_dict['statuses']] return project + @staticmethod + def _attr_to_id(availables, text): + if not text: + return None + + if isinstance(text, basestring) and text.isdigit(): + return text + + for value, key in availables: + if key.lower() == text.lower(): + return value + + return text + def iter_issues(self, query): """ Iter issues with optionnal patterns. @@ -136,13 +151,13 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection): @param query [Query] @return [iter(Issue)] issues """ - # TODO link between text and IDs. + params = self.browser.get_project(query.project) kwargs = {'subject': query.title, - 'author_id': query.author, - 'assigned_to_id': query.assignee, - 'fixed_version_id': query.version, - 'category_id': query.category, - 'status_id': query.status, + 'author_id': self._attr_to_id(params['members'], query.author), + 'assigned_to_id': self._attr_to_id(params['members'], query.assignee), + 'fixed_version_id': self._attr_to_id(params['versions'], query.version), + 'category_id': self._attr_to_id(params['categories'], query.category), + 'status_id': self._attr_to_id(params['statuses'], query.status), } r = self.browser.query_issues(query.project, **kwargs) project = self._build_project(r['project']) @@ -152,6 +167,8 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection): obj.title = issue['subject'] obj.creation = issue['created_on'] obj.updated = issue['updated_on'] + obj.start = issue['start_date'] + obj.due = issue['due_date'] if isinstance(issue['author'], tuple): obj.author = project.find_user(*issue['author']) @@ -162,6 +179,7 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection): else: obj.assignee = issue['assigned_to'] + obj.tracker = issue['tracker'] obj.category = issue['category'] if issue['fixed_version'] is not None: @@ -169,6 +187,7 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection): else: obj.version = None obj.status = project.find_status(issue['status']) + obj.priority = issue['priority'] yield obj def get_issue(self, issue): @@ -189,6 +208,8 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection): issue.body = params['body'] issue.creation = params['created_on'] issue.updated = params['updated_on'] + issue.start = params['start_date'] + issue.due = params['due_date'] issue.fields = {} for key, value in params['fields'].iteritems(): issue.fields[key] = value @@ -214,9 +235,11 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection): issue.history.append(update) issue.author = issue.project.find_user(*params['author']) issue.assignee = issue.project.find_user(*params['assignee']) + issue.tracker = params['tracker'][1] issue.category = params['category'][1] issue.version = issue.project.find_version(*params['version']) issue.status = issue.project.find_status(params['status'][1]) + issue.priority = params['priority'][1] return issue @@ -239,8 +262,12 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection): kwargs = {'title': issue.title, 'version': issue.version.id if issue.version else None, 'assignee': issue.assignee.id if issue.assignee else None, + 'tracker': issue.tracker if issue.tracker else None, 'category': issue.category, 'status': issue.status.id if issue.status else None, + 'priority': issue.priority if issue.priority else None, + 'start': issue.start if issue.start else None, + 'due': issue.due if issue.due else None, 'body': issue.body, 'fields': issue.fields, } diff --git a/modules/redmine/browser.py b/modules/redmine/browser.py index 25de46be..db07b2a8 100644 --- a/modules/redmine/browser.py +++ b/modules/redmine/browser.py @@ -199,7 +199,11 @@ class RedmineBrowser(BaseBrowser): fields = {} for key, div in self.page.iter_custom_fields(): - fields[key] = div.attrib['value'] + if 'value' in div.attrib: + fields[key] = div.attrib['value'] + else: + olist = div.xpath('.//option[@selected="selected"]') + fields[key] = ', '.join({i.attrib['value'] for i in olist}) return fields diff --git a/modules/redmine/pages/issues.py b/modules/redmine/pages/issues.py index a0d2109c..48643969 100644 --- a/modules/redmine/pages/issues.py +++ b/modules/redmine/pages/issues.py @@ -23,8 +23,10 @@ import datetime from weboob.capabilities.bugtracker import IssueError from weboob.tools.browser import BasePage, BrokenPageError +from weboob.tools.date import parse_french_date from weboob.tools.misc import to_unicode from weboob.tools.mech import ClientForm +from weboob.tools.json import json class BaseIssuePage(BasePage): @@ -48,6 +50,10 @@ class BaseIssuePage(BasePage): int(m.group(4)), int(m.group(5))) + m = re.match('(\d+) (\w+) (\d+) (\d+):(\d+)', text) + if m: + return parse_french_date(text) + self.logger.warning('Unable to parse "%s"' % text) return text @@ -109,6 +115,63 @@ class IssuesPage(BaseIssuePage): 'statuses': 'values_status_id', } + def get_from_js(self, pattern, end, is_list=False): + """ + find a pattern in any javascript text + """ + value = None + for script in self.document.xpath('//script'): + txt = script.text + if txt is None: + continue + + start = txt.find(pattern) + if start < 0: + continue + + while True: + if value is None: + value = '' + else: + value += ',' + value += txt[start+len(pattern):start+txt[start+len(pattern):].find(end)+len(pattern)] + + if not is_list: + break + + txt = txt[start+len(pattern)+txt[start+len(pattern):].find(end):] + + start = txt.find(pattern) + if start < 0: + break + return value + + + def get_project(self, project_name): + project = super(IssuesPage, self).get_project(project_name) + if len(project['statuses']) > 0: + return project + + args = self.get_from_js('var availableFilters = ', ';') + if args is None: + return project + + args = json.loads(args) + def get_values(key): + values = [] + if not key in args: + return values + for key, value in args[key]['values']: + if value.isdigit(): + values.append((value, key)) + return values + + project['members'] = get_values('assigned_to_id') + project['categories'] = get_values('category_id') + project['versions'] = get_values('fixed_version_id') + project['statuses'] = get_values('status_id') + return project + def get_query_method(self): return self.document.xpath('//form[@id="query_form"]')[0].attrib['method'].upper() @@ -167,7 +230,9 @@ class NewIssuePage(BaseIssuePage): return m.group(1) def iter_custom_fields(self): - for div in self.document.xpath('//form//input[starts-with(@id, "issue_custom_field")]'): + for div in self.document.xpath('//form//input[starts-with(@id, "issue_custom_field")]|//form//select[starts-with(@id, "issue_custom_field")]'): + if 'type' in div.attrib and div.attrib['type'] == 'hidden': + continue label = self.document.xpath('//label[@for="%s"]' % div.attrib['id'])[0] yield label.text.strip(), div @@ -192,6 +257,29 @@ class NewIssuePage(BaseIssuePage): except ClientForm.ItemNotFoundError: self.logger.warning('Version not found: %s' % version) + def set_tracker(self, tracker): + if tracker: + select = self.parser.select(self.document.getroot(), 'select#issue_tracker_id', 1) + for option in select.findall('option'): + if option.text and option.text.strip() == tracker: + self.browser['issue[tracker_id]'] = [option.attrib['value']] + return + # value = None + # if len(self.document.xpath('//a[@title="New tracker"]')) > 0: + # value = self.browser.create_tracker(self.get_project_name(), tracker, self.get_authenticity_token()) + # if value: + # control = self.browser.find_control('issue[tracker_id]') + # ClientForm.Item(control, {'name': tracker, 'value': value}) + # self.browser['issue[tracker_id]'] = [value] + # else: + # self.logger.warning('Tracker "%s" not found' % tracker) + self.logger.warning('Tracker "%s" not found' % tracker) + else: + try: + self.browser['issue[tracker_id]'] = [''] + except ClientForm.ControlNotFoundError: + self.logger.warning('Tracker item not found') + def set_category(self, category): if category: select = self.parser.select(self.document.getroot(), 'select#issue_category_id', 1) @@ -209,19 +297,61 @@ class NewIssuePage(BaseIssuePage): else: self.logger.warning('Category "%s" not found' % category) else: - self.browser['issue[category_id]'] = [''] + try: + self.browser['issue[category_id]'] = [''] + except ClientForm.ControlNotFoundError: + self.logger.warning('Category item not found') def set_status(self, status): assert status is not None self.browser['issue[status_id]'] = [str(status)] + def set_priority(self, priority): + if priority: + select = self.parser.select(self.document.getroot(), 'select#issue_priority_id', 1) + for option in select.findall('option'): + if option.text and option.text.strip() == priority: + self.browser['issue[priority_id]'] = [option.attrib['value']] + return + # value = None + # if len(self.document.xpath('//a[@title="New priority"]')) > 0: + # value = self.browser.create_priority(self.get_project_name(), priority, self.get_authenticity_token()) + # if value: + # control = self.browser.find_control('issue[priority_id]') + # ClientForm.Item(control, {'name': priority, 'value': value}) + # self.browser['issue[priority_id]'] = [value] + # else: + # self.logger.warning('Priority "%s" not found' % priority) + self.logger.warning('Priority "%s" not found' % priority) + else: + try: + self.browser['issue[priority_id]'] = [''] + except ClientForm.ControlNotFoundError: + self.logger.warning('Priority item not found') + + def set_start(self, start): + if start is not None: + self.browser['issue[start_date]'] = start.strftime("%Y-%m-%d") + #XXX: else set to "" ? + + def set_due(self, due): + if due is not None: + self.browser['issue[due_date]'] = due.strftime("%Y-%m-%d") + #XXX: else set to "" ? + def set_note(self, message): self.browser['notes'] = message.encode('utf-8') def set_fields(self, fields): for key, div in self.iter_custom_fields(): try: - self.browser[div.attrib['name']] = fields[key] + control = self.browser.find_control(div.attrib['name'], nr=0) + if isinstance(control, ClientForm.TextControl): + control.value = fields[key] + else: + item = control.get(label=fields[key].encode("utf-8"), nr=0) + if item and fields[key] != '': + item.selected = True except KeyError: continue @@ -273,13 +403,31 @@ class IssuePage(NewIssuePage): params['updated_on'] = None params['status'] = self._parse_selection('issue_status_id') + params['priority'] = self._parse_selection('issue_priority_id') params['assignee'] = self._parse_selection('issue_assigned_to_id') + params['tracker'] = self._parse_selection('issue_tracker_id') params['category'] = self._parse_selection('issue_category_id') params['version'] = self._parse_selection('issue_fixed_version_id') + div = self.parser.select(self.document.getroot(), 'input#issue_start_date', 1) + if 'value' in div.attrib: + params['start_date'] = datetime.datetime.strptime(div.attrib['value'], "%Y-%m-%d") + else: + params['start_date'] = None + div = self.parser.select(self.document.getroot(), 'input#issue_due_date', 1) + if 'value' in div.attrib: + params['due_date'] = datetime.datetime.strptime(div.attrib['value'], "%Y-%m-%d") + else: + params['due_date'] = None params['fields'] = {} for key, div in self.iter_custom_fields(): - value = div.attrib['value'] + value = '' + if 'value' in div.attrib: + value = div.attrib['value'] + else: + # XXX: use _parse_selection()? + olist = div.xpath('.//option[@selected="selected"]') + value = ', '.join({i.attrib['value'] for i in olist}) params['fields'][key] = value params['attachments'] = [] @@ -326,14 +474,14 @@ class IssuePage(NewIssuePage): pass else: for li in details.findall('li'): - field = li.find('strong').text.decode('utf-8') + field = li.find('strong').text#.decode('utf-8') i = li.findall('i') new = None last = None if len(i) > 0: if len(i) == 2: last = i[0].text.decode('utf-8') - new = i[-1].text.decode('utf-8') + new = i[-1].text#.decode('utf-8') elif li.find('strike') is not None: last = li.find('strike').find('i').text.decode('utf-8') elif li.find('a') is not None: diff --git a/weboob/applications/boobtracker/boobtracker.py b/weboob/applications/boobtracker/boobtracker.py index 3bb307cc..beeaa10c 100644 --- a/weboob/applications/boobtracker/boobtracker.py +++ b/weboob/applications/boobtracker/boobtracker.py @@ -26,12 +26,14 @@ from smtplib import SMTP import sys import os import re +import unicodedata from weboob.capabilities.base import empty, CapBaseObject from weboob.capabilities.bugtracker import ICapBugTracker, Query, Update, Project, Issue, IssueError from weboob.tools.application.repl import ReplApplication, defaultcount from weboob.tools.application.formatters.iformatter import IFormatter, PrettyFormatter from weboob.tools.misc import html2text +from weboob.tools.date import parse_french_date __all__ = ['BoobTracker'] @@ -63,7 +65,9 @@ class IssueFormatter(IFormatter): result += '\n%s\n\n' % obj.body result += self.format_key('Author', '%s (%s)' % (obj.author.name, obj.creation)) result += self.format_attr(obj, 'status') + result += self.format_attr(obj, 'priority') result += self.format_attr(obj, 'version') + result += self.format_attr(obj, 'tracker') result += self.format_attr(obj, 'category') result += self.format_attr(obj, 'assignee') if hasattr(obj, 'fields') and not empty(obj.fields): @@ -122,8 +126,12 @@ class BoobTracker(ReplApplication): group.add_option('--title') group.add_option('--assignee') group.add_option('--target-version', dest='version') + group.add_option('--tracker') group.add_option('--category') group.add_option('--status') + group.add_option('--priority') + group.add_option('--start') + group.add_option('--due') @defaultcount(10) def do_search(self, line): @@ -246,8 +254,12 @@ class BoobTracker(ReplApplication): ISSUE_FIELDS = (('title', (None, False)), ('assignee', ('members', True)), ('version', ('versions', True)), + ('tracker', (None, False)),#XXX ('category', ('categories', False)), ('status', ('statuses', True)), + ('priority', (None, False)),#XXX + ('start', (None, False)), + ('due', (None, False)), ) def get_list_item(self, objects_list, name): @@ -263,6 +275,12 @@ class BoobTracker(ReplApplication): raise ValueError('"%s" is not found' % name) + def sanitize_key(self, key): + if isinstance(key, str): + key = unicode(key, "utf8") + key = unicodedata.normalize('NFKD', key).encode("ascii", "ignore") + return key.replace(' ', '-').capitalize() + def issue2text(self, issue, backend=None): if backend is not None and 'username' in backend.config: sender = backend.config['username'].get() @@ -285,14 +303,15 @@ class BoobTracker(ReplApplication): if len(objects_list) == 0: continue - output += '%s: %s\n' % (key.capitalize(), value) + output += '%s: %s\n' % (self.sanitize_key(key), value) if list_name is not None: availables = ', '.join(['<%s>' % (o if isinstance(o, basestring) else o.name) for o in objects_list]) - output += 'X-Available-%s: %s\n' % (key.capitalize(), availables) + output += 'X-Available-%s: %s\n' % (self.sanitize_key(key), availables) for key, value in issue.fields.iteritems(): - output += '%s: %s\n' % (key, value or '') + output += '%s: %s\n' % (self.sanitize_key(key), value or '') + # TODO: Add X-Available-* for lists output += '\n%s' % (issue.body or 'Please write your bug report here.') return output @@ -312,17 +331,25 @@ class BoobTracker(ReplApplication): if part[1]: new_value += unicode(part[0], part[1]) else: - new_value += unicode(part[0]) + new_value += part[0].decode('utf-8') value = new_value if is_list_object: objects_list = getattr(issue.project, list_name) value = self.get_list_item(objects_list, value) + # FIXME: autodetect + if key in ['start', 'due']: + if len(value) > 0: + #value = datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + value = parse_french_date(value) + else: + value = None + setattr(issue, key, value) for key in issue.fields.keys(): - value = m.get(key) + value = m.get(self.sanitize_key(key)) if value is not None: issue.fields[key] = value.decode('utf-8') @@ -336,7 +363,7 @@ class BoobTracker(ReplApplication): if charset is not None: content += unicode(s, charset) else: - content += unicode(s) + content += unicode(s, encoding='utf-8') except UnicodeError as e: self.logger.warning('Unicode error: %s' % e) continue @@ -367,7 +394,7 @@ class BoobTracker(ReplApplication): except ValueError as e: if not sys.stdin.isatty(): raise - raw_input("%s -- Press Enter to continue..." % e) + raw_input("%s -- Press Enter to continue..." % unicode(e).encode("utf-8")) continue try: @@ -382,7 +409,7 @@ class BoobTracker(ReplApplication): except IssueError as e: if not sys.stdin.isatty(): raise - raw_input("%s -- Press Enter to continue..." % e) + raw_input("%s -- Press Enter to continue..." % unicode(e).encode("utf-8")) def send_notification(self, email_to, issue): text = """Hi, diff --git a/weboob/capabilities/bugtracker.py b/weboob/capabilities/bugtracker.py index 32bb3f19..ecd95966 100644 --- a/weboob/capabilities/bugtracker.py +++ b/weboob/capabilities/bugtracker.py @@ -39,8 +39,10 @@ class Project(CapBaseObject): name = StringField('Name of the project') members = Field('Members of projects', list) versions = Field('List of versions available for this project', list) + trackers = Field('All trackers', list) categories = Field('All categories', list) statuses = Field('Available statuses for issues', list) + priorities = Field('Available priorities for issues', list) def __init__(self, id, name): CapBaseObject.__init__(self, id) @@ -199,14 +201,18 @@ class Issue(CapBaseObject): body = StringField('Text of issue') creation = DateField('Date when this issue has been created') updated = DateField('Date when this issue has been updated for the last time') + start = DateField('Date when this issue starts') + due = DateField('Date when this issue is due for') attachments = Field('List of attached files', list, tuple) history = Field('History of updates', list, tuple) author = Field('Author of this issue', User) assignee = Field('User assigned to this issue', User) + tracker = StringField('Name of the tracker') category = StringField('Name of the category') version = Field('Target version of this issue', Version) status = Field('Status of this issue', Status) fields = Field('Custom fields (key,value)', dict) + priority = StringField('Priority of the issue') #XXX class Query(CapBaseObject):