[twitter] fix trendy tweets (site changed)

modules/twitter/browser.py

@@ -21,7 +21,7 @@ from weboob.browser import LoginBrowser, URL, need_login
 from weboob.exceptions import BrowserIncorrectPassword
 from weboob.capabilities.messages import Message
 from .pages import LoginPage, LoginErrorPage, ThreadPage, Tweet, TrendsPage,\
-    TimelinePage, HomeTimelinePage, SearchTimelinePage
+    TimelinePage, HomeTimelinePage, SearchTimelinePage, SearchHomePage
 
 
 __all__ = ['TwitterBrowser']
@@ -30,19 +30,22 @@ __all__ = ['TwitterBrowser']
 class TwitterBrowser(LoginBrowser):
     BASEURL = 'https://twitter.com/'
 
+    authenticity_token = None
+
     thread_page = URL(u'(?P<user>.+)/status/(?P<_id>.+)', ThreadPage)
     login_error = URL(u'login/error.+', LoginErrorPage)
     tweet = URL(u'i/tweet/create', Tweet)
-    trends = URL(u'trends', TrendsPage)
+    search_home = URL(u'search-home', SearchHomePage)
+    trends = URL(u'i/trends\?pc=true&show_context=false&src=search-home&k=(?P<token>.*)', TrendsPage)
     search = URL(u'i/search/timeline', SearchTimelinePage)
     profil = URL(u'i/profiles/show/(?P<path>.+)/timeline/with_replies', HomeTimelinePage)
     timeline = URL(u'i/timeline', TimelinePage)
     login = URL(u'', LoginPage)
 
     def do_login(self):
-        self.login.go()
+        self.login.stay_or_go()
 
-        if not self.page.logged:
+        if not self.authenticity_token:
             self.authenticity_token = self.page.login(self.username, self.password)
 
         if not self.page.logged or self.login_error.is_here():
@@ -60,11 +63,14 @@ class TwitterBrowser(LoginBrowser):
         if self.username:
             return self.get_logged_trendy_subject()
         else:
-            return self.trends.open().get_trendy_subjects()
+            return self.trends.open(token="").get_trendy_subjects()
 
-    @need_login
     def get_logged_trendy_subject(self):
-        return self.trends.open().get_trendy_subjects()
+        if not self.authenticity_token:
+            self.do_login()
+
+        trends_token = self.search_home.open().get_trends_token()
+        return self.trends.open(token=trends_token).get_trendy_subjects()
 
     @need_login
     def post(self, thread, message):

modules/twitter/pages.py

@@ -23,7 +23,8 @@ from io import StringIO
 import lxml.html as html
 import urllib
 
-from weboob.browser.pages import HTMLPage, JsonPage, FormNotFound, pagination
+from weboob.tools.json import json
+from weboob.browser.pages import HTMLPage, JsonPage, FormNotFound, pagination, LoggedPage
 from weboob.browser.elements import ListElement, ItemElement, method
 from weboob.browser.filters.standard import CleanText, Format, Regexp, Env, DateTime, Filter
 from weboob.browser.filters.html import Link, Attr
@@ -62,11 +63,14 @@ class TwitterJsonHTMLPage(JsonPage):
 
 class LoginPage(HTMLPage):
     def login(self, login, passwd):
-        form = self.get_form(xpath='//form[@action="https://twitter.com/sessions"]')
+        try:
-        form['session[username_or_email]'] = login
+            form = self.get_form(xpath='//form[@action="https://twitter.com/sessions"]')
-        form['session[password]'] = passwd
+            form['session[username_or_email]'] = login
-        form.submit()
+            form['session[password]'] = passwd
-        return form['authenticity_token']
+            form.submit()
+            return form['authenticity_token']
+        except FormNotFound:
+            return CleanText('(//input[@id="authenticity_token"])[1]/@value')(self.doc)
 
     @property
     def logged(self):
@@ -112,6 +116,12 @@ class ThreadPage(HTMLPage):
             obj_date = DatetimeFromTimestamp(Attr('./div/div[@class="stream-item-header"]/small/a/span | ./div/div[@class="ProfileTweet-authorDetails"]/span/a/span', 'data-time'))
 
 
+class SearchHomePage(HTMLPage):
+    def get_trends_token(self):
+        json_data = CleanText('//input[@id="init-data"]/@value')(self.doc)
+        return json.loads(json_data)['trendsCacheKey']
+
+
 class TrendsPage(TwitterJsonHTMLPage):
 
     @method
@@ -154,7 +164,7 @@ class TimelinePage(TwitterJsonHTMLPage):
                 return u'%s?max_position=%s' % (self.page.url.split('?')[0], self.get_last_id())
 
 
-class HomeTimelinePage(TwitterJsonHTMLPage):
+class HomeTimelinePage(TwitterJsonHTMLPage, LoggedPage):
     @pagination
     @method
     class iter_threads(TimelineListElement):