Commit 658d30e9 authored by Romain Bignon's avatar Romain Bignon Committed by ntome

Browser takes a new parameter 'proxy_headers'

It allows class user to set what headers will be sent to the proxy on the
CONNECT request.
parent db08fb3f
# -*- coding: utf-8 -*-
# Copyright(C) 2019 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import requests
__all__ = ['HTTPAdapter']
class HTTPAdapter(requests.adapters.HTTPAdapter):
def __init__(self, *args, **kwargs):
self._proxy_headers = kwargs.pop('proxy_headers', {})
super(HTTPAdapter, self).__init__(*args, **kwargs)
def add_proxy_header(self, key, value):
self._proxy_headers[key] = value
def update_proxy_headers(self, headers):
self._proxy_headers.update(headers)
def proxy_headers(self, proxy):
headers = super(HTTPAdapter, self).proxy_headers(proxy)
headers.update(self._proxy_headers)
return headers
......@@ -52,6 +52,7 @@ from weboob.tools.log import getLogger
from weboob.tools.compat import basestring, unicode, urlparse, urljoin, urlencode, parse_qsl
from weboob.tools.json import json
from .adapters import HTTPAdapter
from .cookies import WeboobCookieJar
from .exceptions import HTTPNotFound, ClientError, ServerError
from .sessions import FuturesSession
......@@ -87,9 +88,10 @@ class Browser(object):
Check SSL certificates.
"""
PROXIES = None
MAX_RETRIES = 2
"""
Maximum retries on failed requests.
"""
MAX_WORKERS = 10
"""
......@@ -116,7 +118,7 @@ class Browser(object):
return localfile
return os.path.join(os.path.dirname(inspect.getfile(cls)), localfile)
def __init__(self, logger=None, proxy=None, responses_dirname=None, weboob=None):
def __init__(self, logger=None, proxy=None, responses_dirname=None, weboob=None, proxy_headers=None):
self.logger = getLogger('browser', logger)
self.responses_dirname = responses_dirname
self.responses_count = 1
......@@ -126,6 +128,7 @@ class Browser(object):
self.VERIFY = self.asset(self.VERIFY)
self.PROXIES = proxy
self.proxy_headers = proxy_headers or {}
self._setup_session(self.PROFILE)
self.url = None
self.response = None
......@@ -215,13 +218,14 @@ class Browser(object):
# defines a max_retries. It's mandatory in case a server is not
# handling keep alive correctly, like the proxy burp
adapter_kwargs = dict(max_retries=self.MAX_RETRIES)
adapter_kwargs = dict(max_retries=self.MAX_RETRIES,
proxy_headers=self.proxy_headers)
# set connection pool size equal to MAX_WORKERS if needed
if self.MAX_WORKERS > requests.adapters.DEFAULT_POOLSIZE:
adapter_kwargs.update(pool_connections=self.MAX_WORKERS,
pool_maxsize=self.MAX_WORKERS)
session.mount('https://', requests.adapters.HTTPAdapter(**adapter_kwargs))
session.mount('http://', requests.adapters.HTTPAdapter(**adapter_kwargs))
session.mount('https://', HTTPAdapter(**adapter_kwargs))
session.mount('http://', HTTPAdapter(**adapter_kwargs))
if self.TIMEOUT:
session.timeout = self.TIMEOUT
......
......@@ -26,7 +26,7 @@ except ImportError:
ThreadPoolExecutor = None
from requests import Session
from requests.adapters import DEFAULT_POOLSIZE, HTTPAdapter
from requests.adapters import DEFAULT_POOLSIZE
from requests.compat import OrderedDict, cookielib
from requests.cookies import RequestsCookieJar, cookiejar_from_dict
from requests.models import PreparedRequest
......@@ -34,6 +34,8 @@ from requests.sessions import merge_setting
from requests.structures import CaseInsensitiveDict
from requests.utils import get_netrc_auth
from .adapters import HTTPAdapter
def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict):
"""
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment