375 lines
12 KiB
Python
375 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright(C) 2012 Laurent Bachelier
|
|
#
|
|
# This file is part of weboob.
|
|
#
|
|
# weboob is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# weboob is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
from __future__ import absolute_import
|
|
|
|
import re
|
|
from urlparse import urlparse, urljoin
|
|
|
|
try:
|
|
import requests
|
|
if int(requests.__version__.split('.')[0]) < 2:
|
|
raise ImportError()
|
|
except ImportError:
|
|
raise ImportError('Please install python-requests >= 2.0')
|
|
|
|
|
|
from weboob.tools.log import getLogger
|
|
|
|
|
|
# TODO define __all__
|
|
|
|
|
|
class Profile(object):
|
|
"""
|
|
A profile represents the way Browser should act.
|
|
Usually it is to mimic a real browser.
|
|
"""
|
|
|
|
def setup_session(self, session):
|
|
"""
|
|
Change default headers, set up hooks, etc.
|
|
|
|
Warning: Do not enable lzma, bzip or bzip2, sdch encodings
|
|
as python-requests does not support it yet.
|
|
Supported as of 2.2: gzip, deflate, compress.
|
|
In doubt, do not change the default Accept-Encoding header
|
|
of python-requests.
|
|
"""
|
|
raise NotImplementedError()
|
|
|
|
|
|
class Weboob(Profile):
|
|
"""
|
|
It's us!
|
|
Recommended for Weboob-friendly websites only.
|
|
"""
|
|
|
|
def __init__(self, version):
|
|
self.version = version
|
|
|
|
def setup_session(self, session):
|
|
session.headers['User-Agent'] = 'weboob/%s' % self.version
|
|
|
|
|
|
class Firefox(Profile):
|
|
"""
|
|
Try to mimic a specific version of Firefox.
|
|
Ideally, it should follow the current ESR Firefox:
|
|
https://www.mozilla.org/en-US/firefox/organizations/all.html
|
|
Do not change the Firefox version without changing the Gecko one!
|
|
"""
|
|
|
|
def setup_session(self, session):
|
|
"""
|
|
Set up headers for a standard Firefox request
|
|
(except for DNT which isn't on by default but is a good idea).
|
|
|
|
The goal is to be unidentifiable.
|
|
"""
|
|
# Replace all base requests headers
|
|
# https://developer.mozilla.org/en/Gecko_user_agent_string_reference
|
|
# https://bugzilla.mozilla.org/show_bug.cgi?id=572650
|
|
session.headers = {
|
|
'Accept-Language': 'en-us,en;q=0.5',
|
|
'Accept-Encoding': 'gzip, deflate',
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0',
|
|
'DNT': '1'}
|
|
|
|
|
|
class Wget(Profile):
|
|
"""
|
|
Common alternative user agent.
|
|
Some websites will give you a version with less JavaScript.
|
|
Some others could ban you (after all, wget is not a real browser).
|
|
"""
|
|
def __init__(self, version='1.11.4'):
|
|
self.version = version
|
|
|
|
def setup_session(self, session):
|
|
# Don't remove base headers, if websites want to block fake browsers,
|
|
# they will probably block any wget user agent anyway.
|
|
session.headers.update({
|
|
'Accept': '*/*',
|
|
'User-Agent': 'Wget/%s' % self.version})
|
|
|
|
|
|
class BaseBrowser(object):
|
|
"""
|
|
Simple browser class.
|
|
Act like a browser, and don't try to do too much.
|
|
"""
|
|
|
|
PROFILE = Firefox()
|
|
TIMEOUT = 10.0
|
|
REFRESH_MAX = 0.0
|
|
|
|
def __init__(self, logger=None, proxy=None):
|
|
self.logger = getLogger('browser', logger)
|
|
self._setup_session(self.PROFILE)
|
|
if proxy is not None:
|
|
self.session.proxies = {'http': proxy, 'https': proxy}
|
|
self.url = None
|
|
self.response = None
|
|
|
|
def _setup_session(self, profile):
|
|
"""
|
|
Set up a python-requests session for our usage.
|
|
"""
|
|
session = requests.Session()
|
|
|
|
if self.TIMEOUT:
|
|
session.timeout = self.TIMEOUT
|
|
## weboob only can provide proxy and HTTP auth options
|
|
session.trust_env = False
|
|
# TODO connect config['verbose'] to our logger
|
|
|
|
profile.setup_session(session)
|
|
|
|
self.session = session
|
|
|
|
def location(self, url, **kwargs):
|
|
"""
|
|
Like open() but also changes the current URL and response.
|
|
This is the most common method to request web pages.
|
|
|
|
Other than that, has the exact same behavior of open().
|
|
"""
|
|
response = self.open(url, **kwargs)
|
|
self.response = response
|
|
self.url = self.response.url
|
|
return response
|
|
|
|
def open(self, url, referrer=None, allow_redirects=True, **kwargs):
|
|
"""
|
|
Make an HTTP request like a browser does:
|
|
* follow redirects (unless disabled)
|
|
* provide referrers (unless disabled)
|
|
|
|
Unless a `method` is explicitly provided, it makes a GET request,
|
|
or a POST if data is not None,
|
|
An empty `data` (not None, like '' or {}) *will* make a POST.
|
|
|
|
It is a wrapper around session.request().
|
|
All session.request() options are available.
|
|
You should use location() or open() and not session.request(),
|
|
since it has some interesting additions, which are easily
|
|
individually disabled through the arguments.
|
|
|
|
Call this instead of location() if you do not want to "visit" the URL
|
|
(for instance, you are downloading a file).
|
|
|
|
:param url: URL
|
|
:type url: str
|
|
|
|
:param data: POST data
|
|
:type url: str or dict or None
|
|
|
|
:param referrer: Force referrer. False to disable sending it, None for guessing
|
|
:type referrer: str or False or None
|
|
|
|
:rtype: :class:`requests.Response`
|
|
"""
|
|
return self._open(url, referrer, allow_redirects, **kwargs)
|
|
|
|
def _open(self, url, referrer=None, allow_redirects=True, **kwargs):
|
|
req = self.build_request(url, referrer, **kwargs)
|
|
preq = self.session.prepare_request(req)
|
|
|
|
# call python-requests
|
|
response = self.session.send(preq, allow_redirects=allow_redirects)
|
|
|
|
if allow_redirects:
|
|
response = self.handle_refresh(response)
|
|
|
|
return response
|
|
|
|
def build_request(self, url, referrer=None, **kwargs):
|
|
"""
|
|
Does the same job as open(), but returns a Request without
|
|
submitting it.
|
|
This allows further customization to the Request.
|
|
"""
|
|
if isinstance(url, requests.Request):
|
|
req = url
|
|
url = req.url
|
|
else:
|
|
req = requests.Request(url=url, **kwargs)
|
|
|
|
# guess method
|
|
if req.method is None:
|
|
if req.data:
|
|
req.method = 'POST'
|
|
else:
|
|
req.method = 'GET'
|
|
|
|
if referrer is None:
|
|
referrer = self.get_referrer(self.url, url)
|
|
if referrer:
|
|
# Yes, it is a misspelling.
|
|
req.headers.setdefault('Referer', referrer)
|
|
|
|
return req
|
|
|
|
REFRESH_RE = re.compile("^(?P<sleep>[\d\.]+)(; url=[\"']?(?P<url>.*?)[\"']?)?$", re.IGNORECASE)
|
|
|
|
def handle_refresh(self, response):
|
|
"""
|
|
Called by _open, to handle Refresh HTTP header.
|
|
|
|
It only redirect to the refresh URL if the sleep time is inferior to
|
|
REFRESH_MAX.
|
|
"""
|
|
if not 'Refresh' in response.headers:
|
|
return response
|
|
|
|
m = self.REFRESH_RE.match(response.headers['Refresh'])
|
|
if m:
|
|
# XXX perhaps we should not redirect if the refresh url is equal to the current url.
|
|
url = m.groupdict().get('url', None) or response.request.url
|
|
sleep = float(m.groupdict()['sleep'])
|
|
|
|
if sleep <= self.REFRESH_MAX:
|
|
self.logger.debug('Refresh to %s' % url)
|
|
return self._open(url)
|
|
else:
|
|
self.logger.debug('Do not refresh to %s because %s > REFRESH_MAX(%s)' % (url, sleep, self.REFRESH_MAX))
|
|
return response
|
|
|
|
self.logger.warning('Unable to handle refresh "%s"' % response.headers['Refresh'])
|
|
|
|
return response
|
|
|
|
def get_referrer(self, oldurl, newurl):
|
|
"""
|
|
Get the referrer to send when doing a request.
|
|
If we should not send a referrer, it will return None.
|
|
|
|
Reference: https://en.wikipedia.org/wiki/HTTP_referer
|
|
|
|
:param oldurl: Current absolute URL
|
|
:type oldurl: str or None
|
|
|
|
:param newurl: Target absolute URL
|
|
:type newurl: str
|
|
|
|
:rtype: str or None
|
|
"""
|
|
if oldurl is None:
|
|
return None
|
|
old = urlparse(oldurl)
|
|
new = urlparse(newurl)
|
|
# Do not leak secure URLs to insecure URLs
|
|
if old.scheme == 'https' and new.scheme != 'https':
|
|
return None
|
|
# Reloading the page. Usually no referrer.
|
|
if oldurl == newurl:
|
|
return None
|
|
# TODO maybe implement some *optional* privacy features:
|
|
# * do not leak referrer to other domains (often breaks websites)
|
|
# * send a fake referrer (root of the current domain)
|
|
# * never send the referrer
|
|
# Inspired by the RefControl Firefox addon.
|
|
return oldurl
|
|
|
|
|
|
class UrlNotAllowed(Exception):
|
|
pass
|
|
|
|
|
|
class DomainBrowser(BaseBrowser):
|
|
"""
|
|
A browser that handles relative URLs and can have a base URL (usually a domain).
|
|
|
|
For instance self.location('/hello') will get http://weboob.org/hello
|
|
if BASEURL is 'http://weboob.org/'.
|
|
"""
|
|
|
|
BASEURL = None
|
|
"""
|
|
Base URL, e.g. 'http://weboob.org/' or 'https://weboob.org/'
|
|
See absurl().
|
|
"""
|
|
|
|
"""
|
|
URLs allowed to load.
|
|
This can be used to force SSL (if the BASEURL is SSL) or any other leakage.
|
|
Set to True to allow only URLs starting by the BASEURL.
|
|
Set it to a list of allowed URLs if you have multiple allowed URLs.
|
|
More complex behavior is possible by overloading url_allowed()
|
|
"""
|
|
RESTRICT_URL = False
|
|
|
|
def url_allowed(self, url):
|
|
"""
|
|
Checks if we are allowed to visit an URL.
|
|
See RESTRICT_URL.
|
|
|
|
:param url: Absolute URL
|
|
:type url: str
|
|
:rtype: bool
|
|
"""
|
|
if self.BASEURL is None or self.RESTRICT_URL is False:
|
|
return True
|
|
if self.RESTRICT_URL is True:
|
|
return url.startswith(self.BASEURL)
|
|
for restrict_url in self.RESTRICT_URL:
|
|
if url.startswith(restrict_url):
|
|
return True
|
|
return False
|
|
|
|
def absurl(self, uri, base=None):
|
|
"""
|
|
Get the absolute URL, relative to the base URL.
|
|
If BASEURL is None, it will try to use the current URL.
|
|
If base is False, it will always try to use the current URL.
|
|
|
|
:param uri: URI to make absolute. It can be already absolute.
|
|
:type uri: str
|
|
|
|
:param base: Base absolute URL.
|
|
:type base: str or None or False
|
|
|
|
:rtype: str
|
|
"""
|
|
if not base:
|
|
base = self.url
|
|
if base is None or base is True:
|
|
base = self.BASEURL
|
|
return urljoin(base, uri)
|
|
|
|
def open(self, req, *args, **kwargs):
|
|
uri = req.url if isinstance(req, requests.Request) else req
|
|
|
|
url = self.absurl(uri)
|
|
if not self.url_allowed(url):
|
|
raise UrlNotAllowed(url)
|
|
|
|
if isinstance(req, requests.Request):
|
|
req.url = url
|
|
else:
|
|
req = url
|
|
return super(DomainBrowser, self).open(req, *args, **kwargs)
|
|
|
|
def go_home(self):
|
|
"""
|
|
Go to the "home" page, usually the BASEURL.
|
|
"""
|
|
return self.location(self.BASEURL or self.absurl('/'))
|