delete 'remove_html_tags' global function, and create IParser.tocleanstring and IParser.strip abstract methods.

weboob/tools/misc.py

@@ -24,7 +24,6 @@ from dateutil import tz
 from logging import warning
 from time import time, sleep
 from tempfile import gettempdir
-import re
 import os
 import sys
 import traceback
@@ -62,10 +61,6 @@ def get_bytes_size(size, unit_name):
         }
     return float(size * unit_data.get(unit_name, 1))
 
-def remove_html_tags(data):
-    p = re.compile(r'<.*?>')
-    return p.sub(' ', data)
-
 try:
     import html2text as h2t
     h2t.UNICODE_SNOB = 1

weboob/tools/parsers/iparser.py

@@ -18,6 +18,12 @@
 # along with weboob. If not, see <http://www.gnu.org/licenses/>.
 
 
+import re
+
+
+__all__ = ['IParser']
+
+
 class IParser(object):
     def parse(self, data, encoding=None):
         """
@@ -34,3 +40,16 @@ class IParser(object):
         Get HTML string from an element.
         """
         raise NotImplementedError()
+
+    def tocleanstring(self, elem):
+        """
+        Get a clean string from an element.
+        """
+        return self.strip(self.tostring(elem))
+
+    def strip(self, data):
+        """
+        Strip a HTML string.
+        """
+        p = re.compile(r'<.*?>')
+        return p.sub(' ', data).strip()

weboob/tools/parsers/lxmlparser.py

@@ -18,6 +18,7 @@
 # along with weboob. If not, see <http://www.gnu.org/licenses/>.
 
 
+import re
 import lxml.html
 
 from .iparser import IParser
@@ -44,6 +45,16 @@ class LxmlHtmlParser(IParser):
     def tostring(self, element):
         return lxml.html.tostring(element, encoding=unicode)
 
+    def tocleanstring(self, element):
+        txt = element.xpath('text()')   # ['foo ', ' bar']
+        txt = ' '.join(txt)             # 'foo   bar'
+        txt = re.sub('\s+', ' ', txt)   # 'foo bar'
+        return txt.strip()
+
+    def strip(self, s):
+        doc = lxml.html.fromstring(s)   # parse html string
+        return self.tocleanstring(doc)
+
     @classmethod
     def select(cls, element, selector, nb=None, method='cssselect'):
         """