Optional global argument to save responses

By adding -S or --save-responses to an application's arguments, every
response will be saved in /tmp/weboob. This is best used with the
-d/--debug option.
It might be interesting to add the headers to the saved response.

weboob/tools/application/base.py

@@ -138,6 +138,7 @@ class BaseApplication(object):
         logging_options.add_option('-d', '--debug', action='store_true', help='display debug messages')
         logging_options.add_option('-q', '--quiet', action='store_true', help='display only error messages')
         logging_options.add_option('-v', '--verbose', action='store_true', help='display info messages')
+        logging_options.add_option('-S', '--save-responses', action='store_true', help='save every response')
         self._parser.add_option_group(logging_options)
         self._parser.add_option('--shell-completion', action='store_true', help=optparse.SUPPRESS_HELP)
 
@@ -290,6 +291,10 @@ class BaseApplication(object):
             print ' '.join(items)
             sys.exit(0)
 
+        if app.options.save_responses:
+            from weboob.tools.browser import BaseBrowser
+            BaseBrowser.SAVE_RESPONSES = True
+
         if app.options.debug:
             level = logging.DEBUG
         elif app.options.verbose:

weboob/tools/browser/browser.py

@@ -26,6 +26,8 @@ import time
 from logging import warning, debug
 from copy import copy
 from threading import RLock
+import os
+import tempfile
 
 from weboob.tools.parsers import get_parser
 from weboob.tools.decorators import retry
@@ -117,6 +119,7 @@ class BaseBrowser(mechanize.Browser):
         'android': 'Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17',
     }
     USER_AGENT = USER_AGENTS['desktop_firefox']
+    SAVE_RESPONSES = False
 
     # ------ Abstract methods --------------------------------------
 
@@ -260,6 +263,20 @@ class BaseBrowser(mechanize.Browser):
         else:
             return None
 
+    def save_response(self, result):
+        """
+        Save a stream to a temporary file, and log its name.
+        The stream is rewinded after saving.
+        """
+        tmpdir = os.path.join(tempfile.gettempdir(), "weboob")
+        if not os.path.isdir(tmpdir):
+            os.makedirs(tmpdir)
+        fd, path = tempfile.mkstemp(prefix="response", dir=tmpdir)
+        with os.fdopen(fd, 'w') as f:
+            f.write(result.read())
+        debug("Response saved to %s" % path)
+        result.seek(0)
+
     def submit(self, *args, **kwargs):
         """
         Submit the selected form.
@@ -349,6 +366,9 @@ class BaseBrowser(mechanize.Browser):
         debug('[user_id=%s] Went on %s' % (self.username, result.geturl()))
         self.last_update = time.time()
 
+        if self.SAVE_RESPONSES:
+            self.save_response(result)
+
         document = self.get_document(result)
         self.page = pageCls(self, document, result.geturl(), groups=page_groups, group_dict=page_group_dict)
         self.page.on_loaded()