From f5c80141ff05b38be32e304d8146efc7faaf6ce7 Mon Sep 17 00:00:00 2001
From: Laurent Bachelier <laurent@bachelier.name>
Date: Tue, 5 Feb 2013 16:25:10 +0100
Subject: [PATCH] paypal: History support through CSV download

---
 modules/paypal/browser.py | 10 ++++--
 modules/paypal/pages.py   | 66 +++++++++++++++++++++++++++++++++++++--
 2 files changed, 70 insertions(+), 6 deletions(-)

diff --git a/modules/paypal/browser.py b/modules/paypal/browser.py
index f2b60989..4d351579 100644
--- a/modules/paypal/browser.py
+++ b/modules/paypal/browser.py
@@ -19,7 +19,7 @@
 
 
 from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
-from .pages import LoginPage, AccountPage, DownloadHistoryPage
+from .pages import LoginPage, AccountPage, DownloadHistoryPage, SubmitPage, HistoryParser
 
 
 __all__ = ['Paypal']
@@ -29,12 +29,13 @@ class Paypal(BaseBrowser):
     DOMAIN = 'www.paypal.com'
     PROTOCOL = 'https'
     # CERTHASH = '74429081f489cb723a82171a94350913d42727053fc86cf5bf5c3d65d39ec449'
-    ENCODING = None  # refer to the HTML encoding
+    ENCODING = 'UTF-8'  # useful for CSV
     PAGES = {
         '/cgi-bin/\?cmd=_login-run$':             LoginPage,
         '/cgi-bin/\?cmd=_login-submit.+$':        LoginPage,  # wrong login
         '/cgi-bin/webscr\?cmd=_account&nav=0.0$':  AccountPage,
         '/cgi-bin/webscr\?cmd=_history-download&nav=0.3.1$':  DownloadHistoryPage,
+        '/cgi-bin/webscr\?dispatch=[a-z0-9]+$': (SubmitPage, HistoryParser()),
     }
 
     def home(self):
@@ -69,12 +70,15 @@ class Paypal(BaseBrowser):
         return self.page.get_account(_id)
 
     def get_history(self, account):
-        raise NotImplementedError()
+        self.download_history()
+        for transaction in self.page.iter_transactions(account):
+            yield transaction
 
     def download_history(self):
         self.location('/en/cgi-bin/webscr?cmd=_history-download&nav=0.3.1')
         assert self.is_on_page(DownloadHistoryPage)
         self.page.download()
+        return self.page.document
 
     def transfer(self, from_id, to_id, amount, reason=None):
         raise NotImplementedError()
diff --git a/modules/paypal/pages.py b/modules/paypal/pages.py
index 6d7a0139..df1717ad 100644
--- a/modules/paypal/pages.py
+++ b/modules/paypal/pages.py
@@ -22,7 +22,8 @@ import re
 import datetime
 
 from weboob.tools.browser import BasePage, BrokenPageError
-from weboob.capabilities.bank import Account
+from weboob.tools.parsers.csvparser import CsvParser
+from weboob.capabilities.bank import Account, Transaction
 from weboob.tools.capabilities.bank.transactions import FrenchTransaction
 
 __all__ = ['LoginPage', 'AccountPage']
@@ -128,8 +129,67 @@ class DownloadHistoryPage(BasePage):
         self.browser['from_a'] = str(today.month)
         self.browser['from_b'] = str(today.day)
 
-        # only "real" stuff, no cancelled payments
-        self.browser['custom_file_type'] = ['comma_completed']
+        self.browser['custom_file_type'] = ['comma_allactivity']
         self.browser['latest_completed_file_type'] = ['']
 
         self.browser.submit()
+
+
+class SubmitPage(BasePage):
+    """
+    Any result of form submission
+    """
+    def iter_transactions(self, account):
+        csv = self.document
+        for row in csv.drows:
+            # only "real" stuff, no cancelled payments etc.
+            if row['Status'] != 'Completed':
+                continue
+            # we filter accounts by currency
+            if account.get_currency(row['Currency']) != account.currency:
+                continue
+            # does not seem to be a real transaction; duplicates others
+            if row['Type'] == u'Authorization':
+                continue
+
+            trans = Transaction(row['Transaction ID'])
+
+            # silly American locale
+            if re.search(r'\d\.\d\d$', row['Net']):
+                date = datetime.datetime.strptime(row['Date'] + ' ' + row['Time'], "%m/%d/%Y %I:%M:%S %p")
+            else:
+                date = datetime.datetime.strptime(row['Date'] + ' ' + row['Time'], "%d/%m/%Y %H:%M:%S")
+            trans.date = date
+            trans.rdate = date
+
+            line = row['Name']
+            if row['Item Title']:
+                line += u' ' + row['Item Title']
+            if row['Auction Site']:
+                line += u"(" + row['Auction Site'] + u")"
+            trans.raw = line
+            trans.label = row['Name']
+
+            if row['Type'].endswith(u'Credit Card'):
+                trans.type = Transaction.TYPE_CARD
+            elif row['Type'].endswith(u'Payment Sent'):
+                trans.type = Transaction.TYPE_ORDER
+            elif row['Type'] == u'Currency Conversion':
+                trans.type = Transaction.TYPE_BANK
+            else:
+                trans.type = Transaction.TYPE_UNKNOWN
+
+            # Net is what happens after the fee (0 for most users), so what is the most "real"
+            trans.amount = clean_amount(row['Net'])
+            trans._gross = clean_amount(row['Gross'])
+            trans._fees = clean_amount(row['Fee'])
+
+            trans._to = row['To Email Address'] or None
+            trans._from = row['From Email Address'] or None
+
+            yield trans
+
+
+class HistoryParser(CsvParser):
+    HEADER = True
+    FMTPARAMS = {'skipinitialspace': True}