fix compatibility with redmine 2.4, support start/end/tracker/priority (courtesy of François Revol)

modules/redmine/backend.py

@@ -120,7 +120,8 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection):
         raise CollectionNotFound(collection.split_path)
 
     ############# CapBugTracker ###################################################
-    def _build_project(self, project_dict):
+    @classmethod
+    def _build_project(cls, project_dict):
         project = Project(project_dict['name'], project_dict['name'])
         project.members = [User(int(u[0]), u[1]) for u in project_dict['members']]
         project.versions = [Version(int(v[0]), v[1]) for v in project_dict['versions']]
@@ -129,6 +130,20 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection):
         project.statuses = [Status(int(s[0]), s[1], 0) for s in project_dict['statuses']]
         return project
 
+    @staticmethod
+    def _attr_to_id(availables, text):
+        if not text:
+            return None
+
+        if isinstance(text, basestring) and text.isdigit():
+            return text
+
+        for value, key in availables:
+            if key.lower() == text.lower():
+                return value
+
+        return text
+
     def iter_issues(self, query):
         """
         Iter issues with optionnal patterns.
@@ -136,13 +151,13 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection):
         @param  query [Query]
         @return [iter(Issue)] issues
         """
-        # TODO link between text and IDs.
+        params = self.browser.get_project(query.project)
         kwargs = {'subject':          query.title,
-                  'author_id':        query.author,
+                  'author_id':        self._attr_to_id(params['members'], query.author),
-                  'assigned_to_id':   query.assignee,
+                  'assigned_to_id':   self._attr_to_id(params['members'], query.assignee),
-                  'fixed_version_id': query.version,
+                  'fixed_version_id': self._attr_to_id(params['versions'], query.version),
-                  'category_id':      query.category,
+                  'category_id':      self._attr_to_id(params['categories'], query.category),
-                  'status_id':        query.status,
+                  'status_id':        self._attr_to_id(params['statuses'], query.status),
                  }
         r = self.browser.query_issues(query.project, **kwargs)
         project = self._build_project(r['project'])
@@ -152,6 +167,8 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection):
             obj.title = issue['subject']
             obj.creation = issue['created_on']
             obj.updated = issue['updated_on']
+            obj.start = issue['start_date']
+            obj.due = issue['due_date']
 
             if isinstance(issue['author'], tuple):
                 obj.author = project.find_user(*issue['author'])
@@ -162,6 +179,7 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection):
             else:
                 obj.assignee = issue['assigned_to']
 
+            obj.tracker = issue['tracker']
             obj.category = issue['category']
 
             if issue['fixed_version'] is not None:
@@ -169,6 +187,7 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection):
             else:
                 obj.version = None
             obj.status = project.find_status(issue['status'])
+            obj.priority = issue['priority']
             yield obj
 
     def get_issue(self, issue):
@@ -189,6 +208,8 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection):
         issue.body = params['body']
         issue.creation = params['created_on']
         issue.updated = params['updated_on']
+        issue.start = params['start_date']
+        issue.due = params['due_date']
         issue.fields = {}
         for key, value in params['fields'].iteritems():
             issue.fields[key] = value
@@ -214,9 +235,11 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection):
             issue.history.append(update)
         issue.author = issue.project.find_user(*params['author'])
         issue.assignee = issue.project.find_user(*params['assignee'])
+        issue.tracker = params['tracker'][1]
         issue.category = params['category'][1]
         issue.version = issue.project.find_version(*params['version'])
         issue.status = issue.project.find_status(params['status'][1])
+        issue.priority = params['priority'][1]
 
         return issue
 
@@ -239,8 +262,12 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection):
         kwargs = {'title':      issue.title,
                   'version':    issue.version.id if issue.version else None,
                   'assignee':   issue.assignee.id if issue.assignee else None,
+                  'tracker':    issue.tracker if issue.tracker else None,
                   'category':   issue.category,
                   'status':     issue.status.id if issue.status else None,
+                  'priority':   issue.priority if issue.priority else None,
+                  'start':      issue.start if issue.start else None,
+                  'due':        issue.due if issue.due else None,
                   'body':       issue.body,
                   'fields':     issue.fields,
                  }

modules/redmine/browser.py

@@ -199,7 +199,11 @@ class RedmineBrowser(BaseBrowser):
 
         fields = {}
         for key, div in self.page.iter_custom_fields():
-            fields[key] = div.attrib['value']
+            if 'value' in div.attrib:
+                fields[key] = div.attrib['value']
+            else:
+                olist = div.xpath('.//option[@selected="selected"]')
+                fields[key] = ', '.join({i.attrib['value'] for i in olist})
 
         return fields
 

modules/redmine/pages/issues.py

@@ -23,8 +23,10 @@ import datetime
 
 from weboob.capabilities.bugtracker import IssueError
 from weboob.tools.browser import BasePage, BrokenPageError
+from weboob.tools.date import parse_french_date
 from weboob.tools.misc import to_unicode
 from weboob.tools.mech import ClientForm
+from weboob.tools.json import json
 
 
 class BaseIssuePage(BasePage):
@@ -48,6 +50,10 @@ class BaseIssuePage(BasePage):
                                      int(m.group(4)),
                                      int(m.group(5)))
 
+        m = re.match('(\d+) (\w+) (\d+) (\d+):(\d+)', text)
+        if m:
+            return parse_french_date(text)
+
         self.logger.warning('Unable to parse "%s"' % text)
         return text
 
@@ -109,6 +115,63 @@ class IssuesPage(BaseIssuePage):
                       'statuses':   'values_status_id',
                      }
 
+    def get_from_js(self, pattern, end, is_list=False):
+        """
+        find a pattern in any javascript text
+        """
+        value = None
+        for script in self.document.xpath('//script'):
+            txt = script.text
+            if txt is None:
+                continue
+
+            start = txt.find(pattern)
+            if start < 0:
+                continue
+
+            while True:
+                if value is None:
+                    value = ''
+                else:
+                    value += ','
+                value += txt[start+len(pattern):start+txt[start+len(pattern):].find(end)+len(pattern)]
+
+                if not is_list:
+                    break
+
+                txt = txt[start+len(pattern)+txt[start+len(pattern):].find(end):]
+
+                start = txt.find(pattern)
+                if start < 0:
+                    break
+            return value
+
+
+    def get_project(self, project_name):
+        project = super(IssuesPage, self).get_project(project_name)
+        if len(project['statuses']) > 0:
+            return project
+
+        args = self.get_from_js('var availableFilters = ', ';')
+        if args is None:
+            return project
+
+        args = json.loads(args)
+        def get_values(key):
+            values = []
+            if not key in args:
+                return values
+            for key, value in args[key]['values']:
+                if value.isdigit():
+                    values.append((value, key))
+            return values
+
+        project['members'] = get_values('assigned_to_id')
+        project['categories'] = get_values('category_id')
+        project['versions'] = get_values('fixed_version_id')
+        project['statuses'] = get_values('status_id')
+        return project
+
     def get_query_method(self):
         return self.document.xpath('//form[@id="query_form"]')[0].attrib['method'].upper()
 
@@ -167,7 +230,9 @@ class NewIssuePage(BaseIssuePage):
         return m.group(1)
 
     def iter_custom_fields(self):
-        for div in self.document.xpath('//form//input[starts-with(@id, "issue_custom_field")]'):
+        for div in self.document.xpath('//form//input[starts-with(@id, "issue_custom_field")]|//form//select[starts-with(@id, "issue_custom_field")]'):
+            if 'type' in div.attrib and div.attrib['type'] == 'hidden':
+                continue
             label = self.document.xpath('//label[@for="%s"]' % div.attrib['id'])[0]
             yield label.text.strip(), div
 
@@ -192,6 +257,29 @@ class NewIssuePage(BaseIssuePage):
         except ClientForm.ItemNotFoundError:
             self.logger.warning('Version not found: %s' % version)
 
+    def set_tracker(self, tracker):
+        if tracker:
+            select = self.parser.select(self.document.getroot(), 'select#issue_tracker_id', 1)
+            for option in select.findall('option'):
+                if option.text and option.text.strip() == tracker:
+                    self.browser['issue[tracker_id]'] = [option.attrib['value']]
+                    return
+            # value = None
+            # if len(self.document.xpath('//a[@title="New tracker"]')) > 0:
+            #     value = self.browser.create_tracker(self.get_project_name(), tracker, self.get_authenticity_token())
+            # if value:
+            #     control = self.browser.find_control('issue[tracker_id]')
+            #     ClientForm.Item(control, {'name': tracker, 'value': value})
+            #     self.browser['issue[tracker_id]'] = [value]
+            # else:
+            #     self.logger.warning('Tracker "%s" not found' % tracker)
+            self.logger.warning('Tracker "%s" not found' % tracker)
+        else:
+            try:
+                self.browser['issue[tracker_id]'] = ['']
+            except ClientForm.ControlNotFoundError:
+                self.logger.warning('Tracker item not found')
+
     def set_category(self, category):
         if category:
             select = self.parser.select(self.document.getroot(), 'select#issue_category_id', 1)
@@ -209,19 +297,61 @@ class NewIssuePage(BaseIssuePage):
             else:
                 self.logger.warning('Category "%s" not found' % category)
         else:
-            self.browser['issue[category_id]'] = ['']
+            try:
+                self.browser['issue[category_id]'] = ['']
+            except ClientForm.ControlNotFoundError:
+                self.logger.warning('Category item not found')
 
     def set_status(self, status):
         assert status is not None
         self.browser['issue[status_id]'] = [str(status)]
 
+    def set_priority(self, priority):
+        if priority:
+            select = self.parser.select(self.document.getroot(), 'select#issue_priority_id', 1)
+            for option in select.findall('option'):
+                if option.text and option.text.strip() == priority:
+                    self.browser['issue[priority_id]'] = [option.attrib['value']]
+                    return
+            # value = None
+            # if len(self.document.xpath('//a[@title="New priority"]')) > 0:
+            #     value = self.browser.create_priority(self.get_project_name(), priority, self.get_authenticity_token())
+            # if value:
+            #     control = self.browser.find_control('issue[priority_id]')
+            #     ClientForm.Item(control, {'name': priority, 'value': value})
+            #     self.browser['issue[priority_id]'] = [value]
+            # else:
+            #     self.logger.warning('Priority "%s" not found' % priority)
+            self.logger.warning('Priority "%s" not found' % priority)
+        else:
+            try:
+                self.browser['issue[priority_id]'] = ['']
+            except ClientForm.ControlNotFoundError:
+                self.logger.warning('Priority item not found')
+
+    def set_start(self, start):
+        if start is not None:
+            self.browser['issue[start_date]'] = start.strftime("%Y-%m-%d")
+        #XXX: else set to "" ?
+
+    def set_due(self, due):
+        if due is not None:
+            self.browser['issue[due_date]'] = due.strftime("%Y-%m-%d")
+        #XXX: else set to "" ?
+
     def set_note(self, message):
         self.browser['notes'] = message.encode('utf-8')
 
     def set_fields(self, fields):
         for key, div in self.iter_custom_fields():
             try:
-                self.browser[div.attrib['name']] = fields[key]
+                control = self.browser.find_control(div.attrib['name'], nr=0)
+                if isinstance(control, ClientForm.TextControl):
+                    control.value = fields[key]
+                else:
+                    item = control.get(label=fields[key].encode("utf-8"), nr=0)
+                    if item and fields[key] != '':
+                        item.selected = True
             except KeyError:
                 continue
 
@@ -273,13 +403,31 @@ class IssuePage(NewIssuePage):
             params['updated_on'] = None
 
         params['status'] = self._parse_selection('issue_status_id')
+        params['priority'] = self._parse_selection('issue_priority_id')
         params['assignee'] = self._parse_selection('issue_assigned_to_id')
+        params['tracker'] = self._parse_selection('issue_tracker_id')
         params['category'] = self._parse_selection('issue_category_id')
         params['version'] = self._parse_selection('issue_fixed_version_id')
+        div = self.parser.select(self.document.getroot(), 'input#issue_start_date', 1)
+        if 'value' in div.attrib:
+            params['start_date'] = datetime.datetime.strptime(div.attrib['value'], "%Y-%m-%d")
+        else:
+            params['start_date'] = None
+        div = self.parser.select(self.document.getroot(), 'input#issue_due_date', 1)
+        if 'value' in div.attrib:
+            params['due_date'] = datetime.datetime.strptime(div.attrib['value'], "%Y-%m-%d")
+        else:
+            params['due_date'] = None
 
         params['fields'] = {}
         for key, div in self.iter_custom_fields():
-            value = div.attrib['value']
+            value = ''
+            if 'value' in div.attrib:
+                value = div.attrib['value']
+            else:
+                # XXX: use _parse_selection()?
+                olist = div.xpath('.//option[@selected="selected"]')
+                value = ', '.join({i.attrib['value'] for i in olist})
             params['fields'][key] = value
 
         params['attachments'] = []
@@ -326,14 +474,14 @@ class IssuePage(NewIssuePage):
                 pass
             else:
                 for li in details.findall('li'):
-                    field = li.find('strong').text.decode('utf-8')
+                    field = li.find('strong').text#.decode('utf-8')
                     i = li.findall('i')
                     new = None
                     last = None
                     if len(i) > 0:
                         if len(i) == 2:
                             last = i[0].text.decode('utf-8')
-                        new = i[-1].text.decode('utf-8')
+                        new = i[-1].text#.decode('utf-8')
                     elif li.find('strike') is not None:
                         last = li.find('strike').find('i').text.decode('utf-8')
                     elif li.find('a') is not None:

weboob/applications/boobtracker/boobtracker.py

@@ -26,12 +26,14 @@ from smtplib import SMTP
 import sys
 import os
 import re
+import unicodedata
 
 from weboob.capabilities.base import empty, CapBaseObject
 from weboob.capabilities.bugtracker import ICapBugTracker, Query, Update, Project, Issue, IssueError
 from weboob.tools.application.repl import ReplApplication, defaultcount
 from weboob.tools.application.formatters.iformatter import IFormatter, PrettyFormatter
 from weboob.tools.misc import html2text
+from weboob.tools.date import parse_french_date
 
 
 __all__ = ['BoobTracker']
@@ -63,7 +65,9 @@ class IssueFormatter(IFormatter):
         result += '\n%s\n\n' % obj.body
         result += self.format_key('Author', '%s (%s)' % (obj.author.name, obj.creation))
         result += self.format_attr(obj, 'status')
+        result += self.format_attr(obj, 'priority')
         result += self.format_attr(obj, 'version')
+        result += self.format_attr(obj, 'tracker')
         result += self.format_attr(obj, 'category')
         result += self.format_attr(obj, 'assignee')
         if hasattr(obj, 'fields') and not empty(obj.fields):
@@ -122,8 +126,12 @@ class BoobTracker(ReplApplication):
         group.add_option('--title')
         group.add_option('--assignee')
         group.add_option('--target-version', dest='version')
+        group.add_option('--tracker')
         group.add_option('--category')
         group.add_option('--status')
+        group.add_option('--priority')
+        group.add_option('--start')
+        group.add_option('--due')
 
     @defaultcount(10)
     def do_search(self, line):
@@ -246,8 +254,12 @@ class BoobTracker(ReplApplication):
     ISSUE_FIELDS = (('title',    (None,       False)),
                     ('assignee', ('members',  True)),
                     ('version',  ('versions', True)),
+                    ('tracker',  (None,       False)),#XXX
                     ('category', ('categories', False)),
                     ('status',   ('statuses', True)),
+                    ('priority', (None,       False)),#XXX
+                    ('start',    (None,       False)),
+                    ('due',      (None,       False)),
                    )
 
     def get_list_item(self, objects_list, name):
@@ -263,6 +275,12 @@ class BoobTracker(ReplApplication):
 
         raise ValueError('"%s" is not found' % name)
 
+    def sanitize_key(self, key):
+        if isinstance(key, str):
+            key = unicode(key, "utf8")
+        key = unicodedata.normalize('NFKD', key).encode("ascii", "ignore")
+        return key.replace(' ', '-').capitalize()
+
     def issue2text(self, issue, backend=None):
         if backend is not None and 'username' in backend.config:
             sender = backend.config['username'].get()
@@ -285,14 +303,15 @@ class BoobTracker(ReplApplication):
                 if len(objects_list) == 0:
                     continue
 
-            output += '%s: %s\n' % (key.capitalize(), value)
+            output += '%s: %s\n' % (self.sanitize_key(key), value)
             if list_name is not None:
                 availables = ', '.join(['<%s>' % (o if isinstance(o, basestring) else o.name)
                                         for o in objects_list])
-                output += 'X-Available-%s: %s\n' % (key.capitalize(), availables)
+                output += 'X-Available-%s: %s\n' % (self.sanitize_key(key), availables)
 
         for key, value in issue.fields.iteritems():
-            output += '%s: %s\n' % (key, value or '')
+            output += '%s: %s\n' % (self.sanitize_key(key), value or '')
+            # TODO: Add X-Available-* for lists
 
         output += '\n%s' % (issue.body or 'Please write your bug report here.')
         return output
@@ -312,17 +331,25 @@ class BoobTracker(ReplApplication):
                 if part[1]:
                     new_value += unicode(part[0], part[1])
                 else:
-                    new_value += unicode(part[0])
+                    new_value += part[0].decode('utf-8')
             value = new_value
 
             if is_list_object:
                 objects_list = getattr(issue.project, list_name)
                 value = self.get_list_item(objects_list, value)
 
+            # FIXME: autodetect
+            if key in ['start', 'due']:
+                if len(value) > 0:
+                    #value = datetime.strptime(value, "%Y-%m-%d %H:%M:%S")
+                    value = parse_french_date(value)
+                else:
+                    value = None
+
             setattr(issue, key, value)
 
         for key in issue.fields.keys():
-            value = m.get(key)
+            value = m.get(self.sanitize_key(key))
             if value is not None:
                 issue.fields[key] = value.decode('utf-8')
 
@@ -336,7 +363,7 @@ class BoobTracker(ReplApplication):
                         if charset is not None:
                             content += unicode(s, charset)
                         else:
-                            content += unicode(s)
+                            content += unicode(s, encoding='utf-8')
                     except UnicodeError as e:
                         self.logger.warning('Unicode error: %s' % e)
                         continue
@@ -367,7 +394,7 @@ class BoobTracker(ReplApplication):
             except ValueError as e:
                 if not sys.stdin.isatty():
                     raise
-                raw_input("%s -- Press Enter to continue..." % e)
+                raw_input("%s -- Press Enter to continue..." % unicode(e).encode("utf-8"))
                 continue
 
             try:
@@ -382,7 +409,7 @@ class BoobTracker(ReplApplication):
             except IssueError as e:
                 if not sys.stdin.isatty():
                     raise
-                raw_input("%s -- Press Enter to continue..." % e)
+                raw_input("%s -- Press Enter to continue..." % unicode(e).encode("utf-8"))
 
     def send_notification(self, email_to, issue):
         text = """Hi,

weboob/capabilities/bugtracker.py

@@ -39,8 +39,10 @@ class Project(CapBaseObject):
     name =          StringField('Name of the project')
     members =       Field('Members of projects', list)
     versions =      Field('List of versions available for this project', list)
+    trackers =      Field('All trackers', list)
     categories =    Field('All categories', list)
     statuses =      Field('Available statuses for issues', list)
+    priorities =    Field('Available priorities for issues', list)
 
     def __init__(self, id, name):
         CapBaseObject.__init__(self, id)
@@ -199,14 +201,18 @@ class Issue(CapBaseObject):
     body =          StringField('Text of issue')
     creation =      DateField('Date when this issue has been created')
     updated =       DateField('Date when this issue has been updated for the last time')
+    start =         DateField('Date when this issue starts')
+    due =           DateField('Date when this issue is due for')
     attachments =   Field('List of attached files', list, tuple)
     history =       Field('History of updates', list, tuple)
     author =        Field('Author of this issue', User)
     assignee =      Field('User assigned to this issue', User)
+    tracker =       StringField('Name of the tracker')
     category =      StringField('Name of the category')
     version =       Field('Target version of this issue', Version)
     status =        Field('Status of this issue', Status)
     fields =        Field('Custom fields (key,value)', dict)
+    priority =      StringField('Priority of the issue') #XXX
 
 
 class Query(CapBaseObject):