Make CapCollection understandable and useable by humans

* Make the declaration of fct and it in the constructor Collection,
 instead of adding them from the outside
* Add a function to flatten a list containing collection (solves the
 radioob search crash)
* Better display of collections in the "ls" command (and display both id
 and title)
* The "cd" command goes to the root of the path (like the UNIX cd)
* Move the Video object of canalplus in a correct path
* Make Collection iterable
* Add comments to CapCollection
* Cache the result of fct in a Collection; it is only called once
* CollectionNotFound errors can be more explicit by providing a path
* Require utf-8 in collection paths
* Code cleanups

modules/canalplus/backend.py

@@ -27,7 +27,7 @@ from weboob.tools.backend import BaseBackend, BackendConfig
 from weboob.tools.value import Value
 
 from .browser import CanalplusBrowser
-from .pages import CanalplusVideo
+from .video import CanalplusVideo
 
 from weboob.capabilities.collection import ICapCollection
 

modules/canalplus/browser.py

@@ -25,7 +25,8 @@ import lxml.etree
 from weboob.tools.browser import BaseBrowser
 from weboob.tools.browser.decorators import id2url
 
-from .pages import InitPage, CanalplusVideo, VideoPage
+from .pages import InitPage, VideoPage
+from .video import CanalplusVideo
 
 from weboob.capabilities.collection import Collection, CollectionNotFound
 
@@ -85,9 +86,12 @@ class CanalplusBrowser(BaseBrowser):
             if len(path) == 0 or not isinstance(collections, (list, Collection)):
                 return collections
             i = path[0]
-            if i not in [collection.title for collection in collections]:
+            matches = [collection
-                raise CollectionNotFound()
+                        for collection in collections
+                        if collection.id == i or collection.title == i]
+            if not len(matches):
+                raise CollectionNotFound(path)
 
-            return walk_res(path[1:], [collection.children for collection in collections if collection.title == i][0])
+            return walk_res(path[1:], matches[0])
 
         return walk_res(split_path, collections)

modules/canalplus/pages/__init__.py

@@ -18,7 +18,6 @@
 # along with weboob. If not, see <http://www.gnu.org/licenses/>.
 
 from .initpage import InitPage
-from .video import CanalplusVideo
 from .videopage import VideoPage
 
-__all__ = ['InitPage', 'VideoPage',  'CanalplusVideo']
+__all__ = ['InitPage', 'VideoPage']

modules/canalplus/pages/initpage.py

@@ -26,24 +26,24 @@ __all__ = ['InitPage']
 
 
 class InitPage(BasePage):
-    
     def on_loaded(self):
         self.collections = []
 
-        def do(id):
+        def do(_id):
-            self.browser.location("http://service.canal-plus.com/video/rest/getMEAs/cplus/" + id)
+            self.browser.location("http://service.canal-plus.com/video/rest/getMEAs/cplus/%s" % _id)
             return self.browser.page.iter_channel()
 
-        ### Parse liste des channels
+        # Parse the list of channels
         for elem in self.document[2].getchildren():
-            coll = Collection()
+            children = []
             for e in elem.getchildren():
                 if e.tag == "NOM":
-                    coll.title = e.text.strip().encode('utf-8')
+                    _id = e.text.strip()
                 elif e.tag == "SELECTIONS":
                     for select in e:
-                        sub = Collection(title=select[1].text.strip().encode('utf-8'))
+                        sub = Collection(_id=select[0].text,
-                        sub.id = select[0].text
+                                title=select[1].text.strip(),
-                        sub.children = do
+                                fct=do)
-                        coll.appendchild(sub)
+                        children.append(sub)
+            coll = Collection(_id, children=children)
             self.collections.append(coll)

modules/canalplus/pages/videopage.py

@@ -23,7 +23,7 @@ from datetime import datetime
 from weboob.capabilities.base import NotAvailable
 from weboob.tools.capabilities.thumbnail import Thumbnail
 from weboob.tools.browser import BasePage
-from .video import CanalplusVideo
+from ..video import CanalplusVideo
 
 
 __all__ = ['VideoPage']

modules/nova/backend.py

@@ -64,7 +64,7 @@ class NovaBackend(BaseBackend, ICapRadio, ICapCollection):
 
     def iter_resources(self, split_path):
         if len(split_path) > 0:
-            raise CollectionNotFound()
+            raise CollectionNotFound(split_path)
 
         for id in self._RADIOS.iterkeys():
             yield self.get_radio(id)

modules/ouifm/backend.py

@@ -48,7 +48,7 @@ class OuiFMBackend(BaseBackend, ICapRadio, ICapCollection):
 
     def iter_resources(self, split_path):
         if len(split_path) > 0:
-            raise CollectionNotFound()
+            raise CollectionNotFound(split_path)
 
         for id in self._RADIOS.iterkeys():
             yield self.get_radio(id)

modules/radiofrance/backend.py

@@ -159,19 +159,20 @@ class RadioFranceBackend(BaseBackend, ICapRadio, ICapCollection):
 
     def iter_resources(self, split_path):
         if len(split_path) == 1 and split_path[0] == 'francebleu':
-            for id in sorted(self._RADIOS.iterkeys()):
+            for _id in sorted(self._RADIOS.iterkeys()):
-                if id.startswith('fb'):
+                if _id.startswith('fb'):
-                    yield self.get_radio(id)
+                    yield self.get_radio(_id)
         elif len(split_path) == 0:
-            for id in sorted(self._RADIOS.iterkeys()):
+            for _id in sorted(self._RADIOS.iterkeys()):
-                if not id.startswith('fb'):
+                if not _id.startswith('fb'):
-                    yield self.get_radio(id)
+                    yield self.get_radio(_id)
-            yield Collection('francebleu', self.iter_resources('francebleu'))
+            yield Collection('francebleu', 'France Bleu',
+                    children=self.iter_resources(['francebleu']))
         else:
-            raise CollectionNotFound()
+            raise CollectionNotFound(split_path)
 
     def iter_radios_search(self, pattern):
-        for radio in self.iter_resources([]):
+        for radio in self._flatten_resources(self.iter_resources([])):
             if pattern.lower() in radio.title.lower() or pattern.lower() in radio.description.lower():
                 yield radio
 

modules/redmine/backend.py

@@ -94,16 +94,17 @@ class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection):
             return self.browser.get_wiki_preview(project, page, content.content)
 
     ############# CapCollection ###################################################
-    def iter_resources(self, path):
+    def iter_resources(self, split_path):
-        if len(path) == 0:
+        if len(split_path) == 0:
-            return [Collection(project.id) for project in self.iter_projects()]
+            return [Collection(project.id, project.name, fct=self.iter_issues)
+                    for project in self.iter_projects()]
 
-        if len(path) == 1:
+        if len(split_path) == 1:
             query = Query()
-            query.project = unicode(path[0])
+            query.project = unicode(split_path[0])
             return self.iter_issues(query)
 
-        raise CollectionNotFound()
+        raise CollectionNotFound(split_path)
 
 
     ############# CapBugTracker ###################################################

weboob/capabilities/bank.py

@@ -76,7 +76,7 @@ class Transfer(CapBaseObject):
 class ICapBank(ICapCollection):
     def iter_resources(self, split_path):
         if len(split_path) > 0:
-            raise CollectionNotFound()
+            raise CollectionNotFound(split_path)
 
         return self.iter_accounts()
 

weboob/capabilities/collection.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright(C) 2010-2011 Nicolas Duhamel
+# Copyright(C) 2010-2012  Nicolas Duhamel, Laurent Bachelier
 #
 # This file is part of weboob.
 #
@@ -21,44 +21,79 @@ from .base import IBaseCap
 
 __all__ = ['ICapCollection', 'Collection', 'CollectionNotFound']
 
+
 class CollectionNotFound(Exception):
-    def __init__(self, msg=None):
+    def __init__(self, split_path=None):
-        if msg is None:
+        if split_path is not None:
+            msg = 'Collection not found: %s' % '/'.join(split_path)
+        else:
             msg = 'Collection not found'
         Exception.__init__(self, msg)
 
 
 class Children(object):
+    """
+    Dynamic property of a Collection.
+    Returns a list, either by calling a function or because
+    it already has the list.
+    """
     def __get__(self, obj, type=None):
-        if callable(obj._childrenfct):
+        if obj._children is None:
-            return obj._childrenfct(obj.id)
+            if callable(obj._fct):
-        else:
+                obj._children = obj._fct(obj.id)
-            return obj._children
+        return obj._children or []
 
-    def __set__(self, obj, value):
-        obj._childrenfct = value
 
 class Collection(object):
     """
-    _childrenfct
+    Collection of objects.
-    _children
+    Should provide a way to be filled, either by providing the children
-    appendchild
+    right away, or a function. The function will be called once with the id
-    children return iterator
+    as an argument if there were no children provided, but only on demand.
+    It can be found in a list of objects, it indicantes a "folder"
+    you can hop into.
+    id and title should be unicode.
     """
     children = Children()
 
-    def __init__(self, title=None, children=None):
+    def __init__(self, _id=None, title=None, children=None, fct=None):
+        self.id = _id
         self.title = title
-        self._children = children if children else []
+        # It does not make sense to have both at init
-        self._childrenfct = None
+        assert not (fct is not None and children is not None)
+        self._children = children
+        self._fct = fct
 
-    def appendchild(self, child):
+    def __iter__(self):
-        self._children.append(child)
+        return iter(self.children)
 
+    def __unicode__(self):
+        if self.title and self.id:
+            return u'%s (%s)' % (self.id, self.title)
+        elif self.id:
+            return u'%s' % self.id
+        else:
+            return u'Unknown collection'
 
-class Ressource(object):
-    pass
 
 class ICapCollection(IBaseCap):
+    def _flatten_resources(self, resources, clean_only=False):
+        """
+        Expand all collections in a list
+        If clean_only is True, do not expand collections, only remove them.
+        """
+        lst = list()
+        for resource in resources:
+            if isinstance(resource, (list, Collection)):
+                if not clean_only:
+                    lst.extend(self._flatten_resources(resource))
+            else:
+                lst.append(resource)
+        return lst
+
     def iter_resources(self, split_path):
+        """
+        split_path is a list, either empty (root path) or with one or many
+        components.
+        """
         raise NotImplementedError()

weboob/tools/application/repl.py

@@ -854,6 +854,14 @@ class ReplApplication(Cmd, ConsoleApplication):
         for obj in self.objects:
             if isinstance(obj, CapBaseObject):
                 self.format(obj)
+            elif isinstance(obj, Collection):
+                if obj.id and obj.title:
+                    print u'Collection: %s%s%s (%s)' % \
+                        (self.BOLD, obj.id, self.NC, obj.title)
+                elif obj.id:
+                    print u'Collection: %s%s%s' % (self.BOLD, obj.id, self.NC)
+                else:
+                    print obj
             else:
                 print obj.title
 
@@ -861,12 +869,14 @@ class ReplApplication(Cmd, ConsoleApplication):
 
     def do_cd(self, line):
         """
-        cd PATH
+        cd [PATH]
 
         Follow a path.
+        If empty, return home.
         """
-        line = line.encode('utf-8')
+        if not len(line.strip()):
-
+            self.working_path.home()
+        else:
             self.working_path.extend(line)
 
         objects = self._fetch_objects()

weboob/tools/path.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright(C) 2010-2011 Nicolas Duhamel
+# Copyright(C) 2010-2012 Nicolas Duhamel, Laurent Bachelier
 #
 # This file is part of weboob.
 #
@@ -27,6 +27,9 @@ class Path(object):
 
 
     def extend(self, user_input):
+        """
+        Add a new part to the current path
+        """
 
         user_input = urllib.quote_plus(user_input)
         user_input = posixpath.normpath(user_input)
@@ -49,8 +52,17 @@ class Path(object):
         self._working_path = final_parse
 
     def restore(self):
+        """
+        Go to the previous path
+        """
         self._working_path = self._previous
 
+    def home(self):
+        """
+        Go to the root
+        """
+        self._previous = self._working_path
+        self._working_path = []
 
     def get(self):
         return copy.copy(self._working_path)