get real amount accounting paypal's commissions

For pro accounts, using bookkeeping tab instead of activity
When there's a commission, transaction's label gets enriched in parenthesis at the end with the actual amount before the fee was deduced.

Signed-off-by: RouxRC <b.ooghe@gmail.com>

modules/paypal/browser.py

@@ -54,7 +54,7 @@ class Paypal(Browser):
         'https://\w+.paypal.com/myaccount/\?nav=0.0': HomePage,
         'https://\w+.paypal.com/businessexp/money': AccountPage,
         'https://\w+.paypal.com/businessexp/summary': ProHistoryPage,
-        'https://\w+.paypal.com/webapps/business/activity\?.*': ProHistoryPage,
+        'https://\w+.paypal.com/webapps/business/bookkeeping\?.*': ProHistoryPage,
         'https://\w+.paypal.com/myaccount/activity/.*': (PartHistoryPage, 'json'),
         'https://\w+.paypal.com/myaccount/': HomePage,
     }
@@ -161,7 +161,7 @@ class Paypal(Browser):
             for trans in chunk:
                 yield trans
 
-    def download_history(self, start, end):
+    def download_history(self, start, end, retry=3):
         """
         Download history.
         However, it is not normalized, and sometimes the download is refused
@@ -172,9 +172,15 @@ class Paypal(Browser):
         # Settings a big magic number so we hope to get all transactions for the period
         LIMIT = '9999'
         if self.account_type == "pro":
-            self.location('https://www.paypal.com/webapps/business/activity?fromdate=' + s + '&todate=' + e + '&transactiontype=ALL_TRANSACTIONS&currency=ALL_TRANSACTIONS_CURRENCY&limit=' + LIMIT)
+            self.location('https://www.paypal.com/webapps/business/bookkeeping?fromdate=' + s + '&todate=' + e + '&transactiontype=BALANCE_AFFECTING_TRANSACTIONS&currency=ALL_TRANSACTIONS_CURRENCY&limit=' + LIMIT)
         else:
             self.location('https://www.paypal.com/myaccount/activity/filter?typeFilter=all&isNewSearch=true&startDate=' + s + '&endDate=' + e + '&limit=' + LIMIT)
+        # catch occasional errors and try to rerun page three times until fail
+        if not self.page:
+            if retry:
+                return self.download_history(start, end, retry=retry-1)
+            self.logger.warning("Error getting history from %s to %s" % (start, end))
+            return False
         return self.page.transaction_left()
 
     def transfer(self, from_id, to_id, amount, reason=None):

modules/paypal/pages.py

@@ -116,17 +116,24 @@ class ProHistoryPage(Page):
                 yield trans
 
     def parse(self):
-        for tr in self.document.xpath('//tr'):
+        for tr in self.document.xpath('//tbody/tr'):
-            t = FrenchTransaction(tr.xpath('./td[@class="transactionId"]/span')[0].text.strip())
+            tlink = tr.xpath('./td[@class="desc"]/a[@class="rowClick"]')[0].attrib['href'].strip()
+            t = FrenchTransaction(tlink[tlink.find('&id=')+4:])
             date = parse_french_date(tr.xpath('./td[@class="date"]')[0].text.strip())
-            status = tr.xpath('./td[@class="desc"]/ul/li[@class="first"]')[0].text.strip()
+            raw = tr.xpath('./td[@class="desc"]/a[@class="rowClick"]')[0].tail.strip()
-            #We pass this because it's not transaction
+            # Filter lines that do not actually modify the balance
-            if status in [u'Créé', u'Annulé', u'Suspendu', u'Mis à jour', u'Actif']:
+            if raw.startswith('Autorisation ') or raw.endswith(' en attente  par PayPal'):
                 continue
-            raw = tr.xpath('./td[@class="desc"]/strong')[0].text.strip()
             t.parse(date=date, raw=raw)
-            amount = tr.xpath('./td[@class="price"]/span')[0].text.strip()
+
+            amount = tr.xpath('./td[@class="price-value net"]')[0].text.strip()
             t.set_amount(amount)
+            commission = tr.xpath('./td[@class="price-value fee"]')[0].text.strip()
+            t.commission = Decimal(t.clean_amount(commission))
+            t.label = t.raw
+            if t.commission:
+                t.label += " (%s)" % tr.xpath('./td[@class="price-value gross"]')[0].text.strip()
+
             t._currency = Account.get_currency(amount)
             yield t