[feedly] New module feedly

modules/feedly/__init__.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2014      Bezleputh
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from .backend import FeedlyBackend
+
+
+__all__ = ['FeedlyBackend']

modules/feedly/backend.py

@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2014      Bezleputh
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from weboob.tools.backend import BaseBackend, BackendConfig
+from weboob.capabilities.collection import ICapCollection
+from weboob.capabilities.messages import ICapMessages, Message, Thread
+from weboob.tools.value import Value, ValueBackendPassword
+
+from .browser import FeedlyBrowser
+from .google import GoogleBrowser
+
+__all__ = ['FeedlyBackend']
+
+
+class FeedlyBackend(BaseBackend, ICapMessages, ICapCollection):
+    NAME = 'feedly'
+    DESCRIPTION = u'handle the popular RSS reading service Feedly'
+    MAINTAINER = u'Bezleputh'
+    EMAIL = 'carton_ben@yahoo.fr'
+    LICENSE = 'AGPLv3+'
+    VERSION = '0.j'
+    STORAGE = {'seen': []}
+    CONFIG = BackendConfig(Value('username', label='Username', default=''),
+                           ValueBackendPassword('password', label='Password', default=''))
+
+    BROWSER = FeedlyBrowser
+
+    def iter_resources(self, objs, split_path):
+        collection = self.get_collection(objs, split_path)
+        if collection.path_level == 0:
+            return self.browser.get_categories()
+
+        if collection.path_level == 1:
+            return self.browser.get_feeds(split_path[0])
+
+        if collection.path_level == 2:
+            url = self.browser.get_feed_url(split_path[0], split_path[1])
+            threads = []
+            for article in self.browser.get_unread_feed(url):
+                thread = self.get_thread(article.id, article)
+                threads.append(thread)
+            return threads
+
+    def validate_collection(self, objs, collection):
+        if collection.path_level in [0, 1, 2]:
+            return
+
+    def get_thread(self, id, entry=None):
+        if isinstance(id, Thread):
+            thread = id
+            id = thread.id
+        else:
+            thread = Thread(id)
+        if entry is None:
+            url = id.split('#')[0]
+            for article in self.browser.get_unread_feed(url):
+                if article.id == id:
+                    entry = article
+        if entry is None:
+            return None
+
+        if not thread.id in self.storage.get('seen', default=[]):
+            entry.flags = Message.IS_UNREAD
+
+        entry.thread = thread
+        thread.title = entry.title
+        thread.root = entry
+        return thread
+
+    def iter_unread_messages(self):
+        for thread in self.iter_threads():
+            for m in thread.iter_all_messages():
+                if m.flags & m.IS_UNREAD:
+                    yield m
+
+    def iter_threads(self):
+        for article in self.browser.iter_threads():
+            yield self.get_thread(article.id, article)
+
+    def set_message_read(self, message):
+        self.browser.set_message_read(message.thread.id.split('#')[-1])
+        self.storage.get('seen', default=[]).append(message.thread.id)
+        self.storage.save()
+
+    def fill_thread(self, thread, fields):
+        return self.get_thread(thread)
+
+    def create_default_browser(self):
+        username = self.config['username'].get()
+        if username:
+            password = self.config['password'].get()
+            login_browser = GoogleBrowser(username, password,
+                                          'https://feedly.com/v3/auth/callback&scope=profile+email&state=Ak7fo397ImkiOiJmZWVkbHkiLCJyIjoiaHR0cDovL2ZlZWRseS5jb20vZmVlZGx5Lmh0bWwiLCJwIjoiR29vZ)2xlUGx1cyIsImMiOiJmZWVkbHkuZGVza3RvcCAyMC40Ljc3NSJ9')
+        else:
+            password = None
+            login_browser = None
+        return self.create_browser(username, password, login_browser)
+
+    OBJECTS = {Thread: fill_thread}

modules/feedly/browser.py

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2014      Bezleputh
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+import simplejson
+
+from weboob.capabilities.collection import Collection
+from weboob.tools.browser2 import LoginBrowser, URL, need_login
+from .pages import EssentialsPage, TokenPage, ContentsPage, PreferencesPage
+
+
+__all__ = ['FeedlyBrowser']
+
+
+class FeedlyBrowser(LoginBrowser):
+    BASEURL = 'http://www.feedly.com'
+
+    essentials = URL('http://s3.feedly.com/essentials/essentials_fr.json', EssentialsPage)
+    token = URL('v3/auth/token', TokenPage)
+    contents = URL('v3/streams/contents', ContentsPage)
+    preferences = URL('v3/preferences', PreferencesPage)
+    marker = URL('v3/markers')
+
+    def __init__(self, username, password, login_browser, *args, **kwargs):
+        super(FeedlyBrowser, self).__init__(username, password, *args, **kwargs)
+        self.login_browser = login_browser
+        self.user_id = None
+
+    def do_login(self):
+        if self.login_browser.code is None or self.user_id is None:
+            self.login_browser.do_login()
+            params = {'code': self.login_browser.code,
+                      'client_id': 'feedly',
+                      'client_secret': '0XP4XQ07VVMDWBKUHTJM4WUQ',
+                      'redirect_uri': 'http://dev.feedly.com/feedly.html',
+                      'grant_type': 'authorization_code'}
+
+            token, self.user_id = self.token.go(data=params).get_token()
+            self.session.headers['X-Feedly-Access-Token'] = token
+
+    @need_login
+    def iter_threads(self):
+        params = {'streamId': 'user/%s/category/global.all' % self.user_id,
+                  'unreadOnly': 'true',
+                  'ranked': 'newest',
+                  'count': '100'}
+        return self.contents.go(params=params).get_articles()
+
+    def get_unread_feed(self, url):
+        params = {'streamId': url,
+                  'backfill': 'true',
+                  'boostMustRead': 'true',
+                  'unreadOnly': 'true'}
+        return self.contents.go(params=params).get_articles()
+
+    def get_categories(self):
+        if self.username is not None and self.password is not None:
+            return self.get_logged_categories()
+        return self.essentials.go().get_categories()
+
+    @need_login
+    def get_logged_categories(self):
+        user_categories = list(self.preferences.go().get_categories())
+        user_categories.append(Collection(['global.saved'], 'Saved'))
+        return user_categories
+
+    def get_feeds(self, category):
+        if self.username is not None and self.password is not None:
+            return self.get_logged_feeds(category)
+        return self.essentials.go().get_feeds(category)
+
+    @need_login
+    def get_logged_feeds(self, category):
+        if category == 'global.saved':
+            type = 'tag'
+        else:
+            type = 'category'
+        url = 'user/%s/%s/%s' % (self.user_id, type, category)
+        return self.get_unread_feed(url)
+
+    def get_feed_url(self, category, feed):
+        return self.essentials.go().get_feed_url(category, feed)
+
+    @need_login
+    def set_message_read(self, _id):
+        datas = {'action': 'markAsRead',
+                 'type': 'entries',
+                 'entryIds': [_id]}
+        self.marker.open(data=simplejson.dumps(datas))

modules/feedly/google.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2014      Bezleputh
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from urlparse import urlparse, parse_qs
+
+from weboob.tools.browser2 import LoginBrowser, URL, HTMLPage
+from weboob.tools.exceptions import BrowserIncorrectPassword
+
+__all__ = ['GoogleBrowser', 'GoogleLoginPage']
+
+
+class GoogleLoginPage(HTMLPage):
+    def login(self, login, passwd):
+        form = self.get_form('//form[@id="gaia_loginform"]')
+        form['Email'] = login
+        form['Passwd'] = passwd
+        form.submit()
+
+
+class GoogleBrowser(LoginBrowser):
+    BASEURL = 'https://accounts.google.com'
+
+    code = None
+    google_login = URL('https://accounts.google.com/(?P<auth>.+)', GoogleLoginPage)
+
+    def __init__(self, username, password, redirect_uri, *args, **kwargs):
+        super(GoogleBrowser, self).__init__(username, password, *args, **kwargs)
+        self.redirect_uri = redirect_uri
+
+    def do_login(self):
+        params = {'response_type': 'code',
+                  'client_id': '534890559860-r6gn7e3agcpiriehe63dkeus0tpl5i4i.apps.googleusercontent.com',
+                  'redirect_uri': self.redirect_uri}
+
+        queryString = "&".join([key+'='+value for key, value in params.items()])
+        self.google_login.go(auth='o/oauth2/auth', params=queryString).login(self.username, self.password)
+
+        try:
+            self.code = parse_qs(urlparse(self.url).query).get('code')[0]
+        except:
+            raise BrowserIncorrectPassword()

modules/feedly/pages.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2014      Bezleputh
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from datetime import datetime
+
+from weboob.capabilities.messages import Message
+from weboob.capabilities.collection import Collection
+from weboob.tools.browser2.page import JsonPage, ListElement, method, ItemElement
+from weboob.tools.browser2.filters import CleanText, Dict, Format, CleanHTML
+
+__all__ = ['TokenPage', 'ContentsPage', 'PreferencesPage']
+
+
+class DictElement(ListElement):
+    def find_elements(self):
+        if self.item_xpath is not None:
+            for el in self.el.get(self.item_xpath):
+                yield el
+        else:
+            yield self.el
+
+
+class ContentsPage(JsonPage):
+
+    @method
+    class get_articles(DictElement):
+        item_xpath = 'items'
+
+        class item(ItemElement):
+            klass = Message
+
+            obj_id = Format(u'%s#%s', CleanText(Dict('origin/streamId')), CleanText(Dict('id')))
+            obj_sender = CleanText(Dict('author', default=u''))
+            obj_title = Format(u'%s - %s', CleanText(Dict('origin/title', default=u'')), CleanText(Dict('title')))
+
+            def obj_date(self):
+                return datetime.fromtimestamp(Dict('published')(self.el) / 1e3)
+
+            def obj_content(self):
+                if 'content' in self.el.keys():
+                    return Format(u'%s%s\r\n',
+                                  CleanHTML(Dict('content/content')), CleanText(Dict('origin/htmlUrl')))(self.el)
+                elif 'summary' in self.el.keys():
+                    return Format(u'%s%s\r\n',
+                                  CleanHTML(Dict('summary/content')), CleanText(Dict('origin/htmlUrl')))(self.el)
+                else:
+                    return ''
+
+
+class TokenPage(JsonPage):
+    def get_token(self):
+        return self.doc['access_token'], self.doc['id']
+
+
+class EssentialsPage(JsonPage):
+    def get_categories(self):
+        for category in self.doc:
+            name = u'%s' % category.get('label')
+            yield Collection([name], name)
+
+    def get_feeds(self, label):
+        for category in self.doc:
+            if category.get('label') == label:
+                feeds = category.get('subscriptions')
+                for feed in feeds:
+                    yield Collection([label, feed.get('title')])
+
+    def get_feed_url(self, _category, _feed):
+        for category in self.doc:
+            if category.get('label') == _category:
+                feeds = category.get('subscriptions')
+                for feed in feeds:
+                    if feed.get('title') == _feed:
+                        return feed.get('id')
+
+
+class PreferencesPage(JsonPage):
+    def get_categories(self):
+        for category, value in self.doc.items():
+            if value in [u"shown", u"hidden"]:
+                yield Collection([category])

modules/feedly/test.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2014      Bezleputh
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from nose.plugins.skip import SkipTest
+from weboob.tools.test import BackendTest
+
+
+class FeedlyTest(BackendTest):
+    BACKEND = 'feedly'
+
+    def test_feedly(self):
+        if self.backend.browser.username:
+            l1 = list(self.backend.iter_threads())
+            assert len(l1)
+            thread = self.backend.get_thread(l1[0].id)
+            assert len(thread.root.content)
+        else:
+            raise SkipTest("User credentials not defined")