nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Make Collection more safe and sane

Browse source 
* Remove callbacks in Collection object
  Make Collection a "dumb" object (and also a base object,
  though it isn't very useful for now)
* Rename Path to WorkingPath, because it is more about managing state
  than being a single path.
* Rewrite almost all WorkingPath, because the code was overly
  complicated for no reason (I tried some special cases and it turned
  out that fromstring didn't handle them, and that the
  quote-escape-unquote was just unecessary). I also rewrote it to be
  more pythonic (no more lambdas and maps) and added tests.
* Require the full split path when creating a Collection. Because, come to
  think of it, an object needs an unique identifier; in the case of
  Collections, it is the full path, not only its last part.
  I might even replace the id by the full split path in the future.
* There is now only one way to get items of a Collection: calling
  iter_resources().
* Rewrite flatten_resources to iter_resources_flat(), which just calls
  iter_resources() recursively.
* Rewrite the collection part of the canalplus module. There is no more
  callback or a page calling the browser to check another page!
  The logic is only in iter_resources().
  The resulting code is not very pretty, but it should get better.
  As a bonus, avoid to reload the main XML file when we already have it
  open.
* change_path() now expects a split path and not a string.
* up/home special cases for "cd" are handled in the same place, and
  store the previous place properly (but are not yet exploitable by
  an user command).

This is a big commit but it would be hard to split it in *working*
commits.

If you read this entire commit message, I will buy you a beer.

refs #774
fixes #773
This commit is contained in:
Laurent Bachelier 2012-03-08 00:33:49 +01:00
commit b4b7182960
13 changed files with 147 additions and 148 deletions
Download patch file Download diff file Expand all files Collapse all files

2
weboob/applications/boobtracker/boobtracker.py
View file

@ -136,7 +136,7 @@ class BoobTracker(ReplApplication):
query.category = self.options.category
query.status = self.options.status
self.change_path('/%s/search' % query.project)
self.change_path([query.project, u'search'])
for backend, issue in self.do('iter_issues', query, backends=backends):
self.add_object(issue)
self.format(issue)

2
weboob/applications/flatboob/flatboob.py
View file

@ -147,7 +147,7 @@ class Flatboob(ReplApplication):
query.cost_max = self.ask_int('Enter max cost')
query.nb_rooms = self.ask_int('Enter number of rooms')
self.change_path('/housings')
self.change_path([u'housings'])
for backend, housing in self.do('search_housings', query):
self.add_object(housing)
self.format(housing)

2
weboob/applications/radioob/radioob.py
View file

@ -135,7 +135,7 @@ class Radioob(ReplApplication):
If PATTERN is not given, this command will list all the radios.
"""
self.set_formatter_header(u'Search pattern: %s' % pattern if pattern else u'All radios')
self.change_path('/search')
self.change_path([u'search'])
for backend, radio in self.do('iter_radios_search', pattern=pattern):
self.add_object(radio)
self.format(radio)

2
weboob/applications/videoob/videoob.py
View file

@ -223,7 +223,7 @@ class Videoob(ReplApplication):
return 1
self.set_formatter_header(u'Search pattern: %s' % pattern if pattern else u'Latest videos')
self.change_path('/search')
self.change_path([u'search'])
for backend, video in self.do('search_videos', pattern=pattern, nsfw=self.nsfw,
max_results=self.options.count):
self.add_object(video)

2
weboob/applications/weboorrents/weboorrents.py
View file

@ -175,7 +175,7 @@ class Weboorrents(ReplApplication):
Search torrents.
"""
self.change_path('/search')
self.change_path([u'search'])
if not pattern:
pattern = None
self.set_formatter_header(u'Search pattern: %s' % pattern if pattern else u'Latest torrents')

4
weboob/applications/wetboobs/wetboobs.py
View file

@ -96,7 +96,7 @@ class WetBoobs(ReplApplication):
Search cities.
"""
self.change_path('/cities')
self.change_path(['cities'])
for backend, city in self.do('iter_city_search', pattern, caps=ICapWeather):
self.add_object(city)
self.format(city)
@ -143,7 +143,7 @@ class WetBoobs(ReplApplication):
List all rivers. If PATTERN is specified, search on a pattern.
"""
self.change_path('/gauges')
self.change_path([u'gauges'])
for backend, gauge in self.do('iter_gauges', pattern or None, caps=ICapWaterLevel):
self.add_object(gauge)
self.format(gauge)

60
weboob/capabilities/collection.py
View file

@ -17,7 +17,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .base import IBaseCap
from .base import IBaseCap, CapBaseObject
__all__ = ['ICapCollection', 'Collection', 'CollectionNotFound']
@ -31,42 +31,18 @@ class CollectionNotFound(Exception):
Exception.__init__(self, msg)
class Children(object):
class Collection(CapBaseObject):
"""
Dynamic property of a Collection.
Returns a list, either by calling a function or because
it already has the list.
"""
def __get__(self, obj, type=None):
if obj._children is None:
if callable(obj._fct):
obj._children = obj._fct(obj.id)
return obj._children or []
A Collection is a "fake" object returned in results, which shows you can get
more results if you go into its path.
class Collection(object):
It is a dumb object, it must not contain callbacks to a backend.
"""
Collection of objects.
Should provide a way to be filled, either by providing the children
right away, or a function. The function will be called once with the id
as an argument if there were no children provided, but only on demand.
It can be found in a list of objects, it indicantes a "folder"
you can hop into.
id and title should be unicode.
"""
children = Children()
backend = None
def __init__(self, _id=None, title=None, children=None, fct=None):
self.id = _id
def __init__(self, split_path, backend=None, title=None):
self.split_path = split_path
self.title = title
# It does not make sense to have both at init
assert not (fct is not None and children is not None)
self._children = children
self._fct = fct
def __iter__(self):
return iter(self.children)
_id = split_path[-1] if len(split_path) else None
CapBaseObject.__init__(self, _id, backend)
def __unicode__(self):
if self.title and self.id:
@ -78,19 +54,19 @@ class Collection(object):
class ICapCollection(IBaseCap):
def _flatten_resources(self, resources, clean_only=False):
def iter_resources_flat(self, objs, split_path, clean_only=False):
"""
Expand all collections in a list
If clean_only is True, do not expand collections, only remove them.
Call iter_resources() to fetch all resources in the tree.
If clean_only is True, do not explore paths, only remove them.
split_path is used to set the starting path.
"""
lst = list()
for resource in resources:
if isinstance(resource, (list, Collection)):
for resource in self.iter_resources(objs, split_path):
if isinstance(resource, Collection):
if not clean_only:
lst.extend(self._flatten_resources(resource))
for res in self.iter_resources_flat(objs, resource.split_path):
yield res
else:
lst.append(resource)
return lst
yield resource
def iter_resources(self, objs, split_path):
"""

24
weboob/tools/application/repl.py
View file

@ -30,7 +30,7 @@ from weboob.capabilities.base import FieldNotFound, CapBaseObject
from weboob.core import CallErrors
from weboob.tools.application.formatters.iformatter import MandatoryFieldsNotFound
from weboob.tools.misc import to_unicode
from weboob.tools.path import Path
from weboob.tools.path import WorkingPath
from weboob.tools.ordereddict import OrderedDict
from weboob.capabilities.collection import Collection, ICapCollection, CollectionNotFound
@ -137,23 +137,22 @@ class ReplApplication(Cmd, ConsoleApplication):
self._interactive = False
self.objects = []
self.collections = []
self.working_path = Path()
self.working_path = WorkingPath()
@property
def interactive(self):
return self._interactive
def _change_prompt(self):
path = self.working_path.tostring()
if len(path) > 0 and path != '/':
self.prompt = '%s:%s> ' % (self.APPNAME, path)
if len(self.working_path.get()):
self.prompt = u'%s:%s> ' % (self.APPNAME, unicode(self.working_path))
else:
self.prompt = '%s> ' % (self.APPNAME)
self.prompt = u'%s> ' % (self.APPNAME)
self.objects = []
self.collections = []
def change_path(self, path):
self.working_path.fromstring(path)
def change_path(self, split_path):
self.working_path.location(split_path)
self._change_prompt()
def add_object(self, obj):
@ -885,16 +884,19 @@ class ReplApplication(Cmd, ConsoleApplication):
cd [PATH]
Follow a path.
If empty, return home.
".." is a special case and goes up one directory.
"" is a special case and goes home.
"""
if not len(line.strip()):
self.working_path.home()
elif line.strip() == '..':
self.working_path.up()
else:
self.working_path.extend(line)
self.working_path.cd1(line)
objects, collections = self._fetch_objects(objs=self.COLLECTION_OBJECTS)
if len(objects) + len(collections) == 0:
print >>sys.stderr, "Path: %s not found" % self.working_path.tostring()
print >>sys.stderr, u"Path: %s not found" % unicode(self.working_path)
self.working_path.restore()
return 1

98
weboob/tools/path.py
View file

@ -16,64 +16,80 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import urllib
import posixpath
import copy
from copy import copy
from posixpath import sep, join
class Path(object):
class WorkingPath(object):
def __init__(self):
self._working_path = []
self._previous = self._working_path
self.split_path = []
self.previous = copy(self.split_path)
def extend(self, user_input):
def cd1(self, user_input):
"""
Add a new part to the current path
Append *one* level to the current path.
This means that separators (/) will get escaped.
"""
split_path = self.get()
split_path.append(user_input)
self.location(split_path)
user_input = urllib.quote_plus(user_input)
user_input = posixpath.normpath(user_input)
escape = lambda s: s.replace('/', '%2F')
current_path = map(escape, self._working_path)
abspath = posixpath.normpath(posixpath.join('/' + '/'.join(current_path), user_input))
abspath = abspath.split('/')[1:]
while len(abspath) > 0 and abspath[0] == u'': del abspath[0]
final_parse = map(urllib.unquote_plus, abspath)
self._previous = self._working_path
if len(final_parse) == 0:
self._working_path = []
self._working_path = final_parse
def location(self, split_path):
"""
Go to a new path, and store the previous path.
"""
self.previous = self.get()
self.split_path = split_path
def restore(self):
"""
Go to the previous path
"""
self._working_path = self._previous
self.split_path, self.previous = self.previous, self.split_path
def home(self):
"""
Go to the root
"""
self._previous = self._working_path
self._working_path = []
self.location([])
def up(self):
"""
Go up one directory
"""
self.location(self.split_path[:-1])
def get(self):
return copy.copy(self._working_path)
"""
Get the current working path
"""
return copy(self.split_path)
def fromstring(self, path):
if path[0] == '/':
path = path[1:]
escape = lambda s: s.replace('\/', '/')
self._working_path = map(escape, path.split('/'))
def __unicode__(self):
return join(sep, *[s.replace(u'/', u'\/') for s in self.split_path])
def tostring(self):
escape = lambda s: s.replace('/', '\/')
path = map(escape, self._working_path)
return '/' + '/'.join(path)
def test():
wp = WorkingPath()
assert wp.get() == []
assert unicode(wp) == u'/'
wp.cd1(u'lol')
assert wp.get() == [u'lol']
assert unicode(wp) == u'/lol'
wp.cd1(u'cat')
assert wp.get() == [u'lol', u'cat']
assert unicode(wp) == u'/lol/cat'
wp.restore()
assert unicode(wp) == u'/lol'
wp.home()
assert wp.get() == []
assert unicode(wp) == u'/'
wp.up()
assert wp.get() == []
assert unicode(wp) == u'/'
wp.location(['aa / aa', 'bbbb'])
assert unicode(wp) == u'/aa \/ aa/bbbb'
wp.up()
assert unicode(wp) == u'/aa \/ aa'
wp.cd1(u'héhé/hé')
assert unicode(wp) == u'/aa \/ aa/héhé\/hé'