From c4f221fba0a5e3d886b6f1daa051729d546a8649 Mon Sep 17 00:00:00 2001 From: Laurent Bachelier Date: Fri, 7 Aug 2015 16:28:19 +0200 Subject: [PATCH] Change pool size after session init FuturesSession does this, but it is lost since the HTTPAdapters are later replaced. --- weboob/browser/browsers.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/weboob/browser/browsers.py b/weboob/browser/browsers.py index 93fcdfdc..f4254880 100644 --- a/weboob/browser/browsers.py +++ b/weboob/browser/browsers.py @@ -196,9 +196,13 @@ class Browser(object): # defines a max_retries. It's mandatory in case a server is not # handling keep alive correctly, like the proxy burp - a = requests.adapters.HTTPAdapter(max_retries=self.MAX_RETRIES) - session.mount('http://', a) - session.mount('https://', a) + adapter_kwargs = dict(max_retries=self.MAX_RETRIES) + # set connection pool size equal to MAX_WORKERS if needed + if self.MAX_WORKERS > requests.adapters.DEFAULT_POOLSIZE: + adapter_kwargs.update(pool_connections=self.MAX_WORKERS, + pool_maxsize=self.MAX_WORKERS) + session.mount('https://', requests.adapters.HTTPAdapter(**adapter_kwargs)) + session.mount('http://', requests.adapters.HTTPAdapter(**adapter_kwargs)) if self.TIMEOUT: session.timeout = self.TIMEOUT