diff --git a/modules/twitter/browser.py b/modules/twitter/browser.py index 5c32fa03..588e33df 100644 --- a/modules/twitter/browser.py +++ b/modules/twitter/browser.py @@ -21,7 +21,7 @@ from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword from weboob.capabilities.messages import Message from .pages import LoginPage, LoginErrorPage, ThreadPage, Tweet, TrendsPage,\ - TimelinePage, HomeTimelinePage, SearchTimelinePage + TimelinePage, HomeTimelinePage, SearchTimelinePage, SearchHomePage __all__ = ['TwitterBrowser'] @@ -30,19 +30,22 @@ __all__ = ['TwitterBrowser'] class TwitterBrowser(LoginBrowser): BASEURL = 'https://twitter.com/' + authenticity_token = None + thread_page = URL(u'(?P.+)/status/(?P<_id>.+)', ThreadPage) login_error = URL(u'login/error.+', LoginErrorPage) tweet = URL(u'i/tweet/create', Tweet) - trends = URL(u'trends', TrendsPage) + search_home = URL(u'search-home', SearchHomePage) + trends = URL(u'i/trends\?pc=true&show_context=false&src=search-home&k=(?P.*)', TrendsPage) search = URL(u'i/search/timeline', SearchTimelinePage) profil = URL(u'i/profiles/show/(?P.+)/timeline/with_replies', HomeTimelinePage) timeline = URL(u'i/timeline', TimelinePage) login = URL(u'', LoginPage) def do_login(self): - self.login.go() + self.login.stay_or_go() - if not self.page.logged: + if not self.authenticity_token: self.authenticity_token = self.page.login(self.username, self.password) if not self.page.logged or self.login_error.is_here(): @@ -60,11 +63,14 @@ class TwitterBrowser(LoginBrowser): if self.username: return self.get_logged_trendy_subject() else: - return self.trends.open().get_trendy_subjects() + return self.trends.open(token="").get_trendy_subjects() - @need_login def get_logged_trendy_subject(self): - return self.trends.open().get_trendy_subjects() + if not self.authenticity_token: + self.do_login() + + trends_token = self.search_home.open().get_trends_token() + return self.trends.open(token=trends_token).get_trendy_subjects() @need_login def post(self, thread, message): diff --git a/modules/twitter/pages.py b/modules/twitter/pages.py index aeb58da1..03a6694f 100644 --- a/modules/twitter/pages.py +++ b/modules/twitter/pages.py @@ -23,7 +23,8 @@ from io import StringIO import lxml.html as html import urllib -from weboob.browser.pages import HTMLPage, JsonPage, FormNotFound, pagination +from weboob.tools.json import json +from weboob.browser.pages import HTMLPage, JsonPage, FormNotFound, pagination, LoggedPage from weboob.browser.elements import ListElement, ItemElement, method from weboob.browser.filters.standard import CleanText, Format, Regexp, Env, DateTime, Filter from weboob.browser.filters.html import Link, Attr @@ -62,11 +63,14 @@ class TwitterJsonHTMLPage(JsonPage): class LoginPage(HTMLPage): def login(self, login, passwd): - form = self.get_form(xpath='//form[@action="https://twitter.com/sessions"]') - form['session[username_or_email]'] = login - form['session[password]'] = passwd - form.submit() - return form['authenticity_token'] + try: + form = self.get_form(xpath='//form[@action="https://twitter.com/sessions"]') + form['session[username_or_email]'] = login + form['session[password]'] = passwd + form.submit() + return form['authenticity_token'] + except FormNotFound: + return CleanText('(//input[@id="authenticity_token"])[1]/@value')(self.doc) @property def logged(self): @@ -112,6 +116,12 @@ class ThreadPage(HTMLPage): obj_date = DatetimeFromTimestamp(Attr('./div/div[@class="stream-item-header"]/small/a/span | ./div/div[@class="ProfileTweet-authorDetails"]/span/a/span', 'data-time')) +class SearchHomePage(HTMLPage): + def get_trends_token(self): + json_data = CleanText('//input[@id="init-data"]/@value')(self.doc) + return json.loads(json_data)['trendsCacheKey'] + + class TrendsPage(TwitterJsonHTMLPage): @method @@ -154,7 +164,7 @@ class TimelinePage(TwitterJsonHTMLPage): return u'%s?max_position=%s' % (self.page.url.split('?')[0], self.get_last_id()) -class HomeTimelinePage(TwitterJsonHTMLPage): +class HomeTimelinePage(TwitterJsonHTMLPage, LoggedPage): @pagination @method class iter_threads(TimelineListElement):