diff --git a/weboob/browser/adapters.py b/weboob/browser/adapters.py new file mode 100644 index 0000000000000000000000000000000000000000..60e0f50cbf305106fa000e711dd162923a874184 --- /dev/null +++ b/weboob/browser/adapters.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Romain Bignon +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with weboob. If not, see . + + +import requests + + +__all__ = ['HTTPAdapter'] + + +class HTTPAdapter(requests.adapters.HTTPAdapter): + def __init__(self, *args, **kwargs): + self._proxy_headers = kwargs.pop('proxy_headers', {}) + super(HTTPAdapter, self).__init__(*args, **kwargs) + + def add_proxy_header(self, key, value): + self._proxy_headers[key] = value + + def update_proxy_headers(self, headers): + self._proxy_headers.update(headers) + + def proxy_headers(self, proxy): + headers = super(HTTPAdapter, self).proxy_headers(proxy) + headers.update(self._proxy_headers) + return headers diff --git a/weboob/browser/browsers.py b/weboob/browser/browsers.py index ed79fcbb432cc4d3a194e1dc1c15893860d09ce5..01e8b19572ef97245d21f47cb644abf0f1443f35 100644 --- a/weboob/browser/browsers.py +++ b/weboob/browser/browsers.py @@ -52,6 +52,7 @@ from weboob.tools.compat import basestring, unicode, urlparse, urljoin, urlencode, parse_qsl from weboob.tools.json import json +from .adapters import HTTPAdapter from .cookies import WeboobCookieJar from .exceptions import HTTPNotFound, ClientError, ServerError from .sessions import FuturesSession @@ -87,9 +88,10 @@ class Browser(object): Check SSL certificates. """ - PROXIES = None - MAX_RETRIES = 2 + """ + Maximum retries on failed requests. + """ MAX_WORKERS = 10 """ @@ -116,7 +118,7 @@ def asset(cls, localfile): return localfile return os.path.join(os.path.dirname(inspect.getfile(cls)), localfile) - def __init__(self, logger=None, proxy=None, responses_dirname=None, weboob=None): + def __init__(self, logger=None, proxy=None, responses_dirname=None, weboob=None, proxy_headers=None): self.logger = getLogger('browser', logger) self.responses_dirname = responses_dirname self.responses_count = 1 @@ -126,6 +128,7 @@ def __init__(self, logger=None, proxy=None, responses_dirname=None, weboob=None) self.VERIFY = self.asset(self.VERIFY) self.PROXIES = proxy + self.proxy_headers = proxy_headers or {} self._setup_session(self.PROFILE) self.url = None self.response = None @@ -215,13 +218,14 @@ def _setup_session(self, profile): # defines a max_retries. It's mandatory in case a server is not # handling keep alive correctly, like the proxy burp - adapter_kwargs = dict(max_retries=self.MAX_RETRIES) + adapter_kwargs = dict(max_retries=self.MAX_RETRIES, + proxy_headers=self.proxy_headers) # set connection pool size equal to MAX_WORKERS if needed if self.MAX_WORKERS > requests.adapters.DEFAULT_POOLSIZE: adapter_kwargs.update(pool_connections=self.MAX_WORKERS, pool_maxsize=self.MAX_WORKERS) - session.mount('https://', requests.adapters.HTTPAdapter(**adapter_kwargs)) - session.mount('http://', requests.adapters.HTTPAdapter(**adapter_kwargs)) + session.mount('https://', HTTPAdapter(**adapter_kwargs)) + session.mount('http://', HTTPAdapter(**adapter_kwargs)) if self.TIMEOUT: session.timeout = self.TIMEOUT diff --git a/weboob/browser/sessions.py b/weboob/browser/sessions.py index 76aca9b7d58e31044c79c4dbd385e1dfa3268969..f6630de4791b6299c1a5aa4b1afdce71e239bc89 100644 --- a/weboob/browser/sessions.py +++ b/weboob/browser/sessions.py @@ -26,7 +26,7 @@ ThreadPoolExecutor = None from requests import Session -from requests.adapters import DEFAULT_POOLSIZE, HTTPAdapter +from requests.adapters import DEFAULT_POOLSIZE from requests.compat import OrderedDict, cookielib from requests.cookies import RequestsCookieJar, cookiejar_from_dict from requests.models import PreparedRequest @@ -34,6 +34,8 @@ from requests.structures import CaseInsensitiveDict from requests.utils import get_netrc_auth +from .adapters import HTTPAdapter + def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict): """