Commit e1a1dcd7 authored by Phyks (Lucas Verney)'s avatar Phyks (Lucas Verney) Committed by Florent Fourcot

Use weboob.tools.compat instead of urllib/urlparse

parent 19e598cb
Pipeline #733 failed
......@@ -18,10 +18,10 @@
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.browser import PagesBrowser, URL
from weboob.tools.compat import quote_plus
from .pages import SearchPage, AdvertPage, AdvertsJsonPage
import urllib
from datetime import date, timedelta
__all__ = ['AdeccoBrowser']
......@@ -50,8 +50,8 @@ class AdeccoBrowser(PagesBrowser):
def advanced_search_job(self, publication_date=0, contract_type=None, conty=None, activity_domain=None,
job='', town=''):
params = self.search_page.go(job=urllib.quote_plus(job.encode('utf-8')),
town=urllib.quote_plus(town.encode('utf-8'))).get_post_params()
params = self.search_page.go(job=quote_plus(job.encode('utf-8')),
town=quote_plus(town.encode('utf-8'))).get_post_params()
if contract_type:
self.page.url += '&employmenttype=%s' % contract_type
......
......@@ -21,7 +21,7 @@
from weboob.browser import PagesBrowser, URL
from .pages import BasePage
from urlparse import urlparse
from weboob.tools.compat import urlparse
import re
......
......@@ -25,11 +25,11 @@ from weboob.capabilities.base import NotAvailable, NotLoaded, find_object
from weboob.capabilities.cinema import Movie, Person
from weboob.browser.browsers import APIBrowser
from weboob.browser.profiles import Android
from weboob.tools.compat import urlencode
import base64
import hashlib
from datetime import datetime, date, timedelta
import time
import urllib
__all__ = ['AllocineBrowser']
......@@ -41,14 +41,18 @@ class AllocineBrowser(APIBrowser):
SECRET_KEY = '29d185d98c984a359e6e6f26a0474269'
def __do_request(self, method, params):
params_encode = urllib.urlencode(params)
sed = time.strftime('%Y%m%d', time.localtime())
sig = base64.b64encode(hashlib.sha1(self.SECRET_KEY + params_encode + '&sed=' + sed).digest())
query_url = 'http://api.allocine.fr/rest/v3/' + method + '?' + params_encode + '&sed=' + sed + '&sig=' + sig
return self.request(query_url)
params["sed"] = time.strftime('%Y%m%d', time.localtime())
params["sig"] = base64.b64encode(
hashlib.sha1(
self.SECRET_KEY +
urlencode(params)
).digest()
)
return self.request(
'http://api.allocine.fr/rest/v3/{}'.format(method),
params=params
)
def iter_movies(self, pattern):
params = [('partner', self.PARTNER_KEY),
......
......@@ -19,7 +19,7 @@
from weboob.browser import PagesBrowser, URL
from .pages import ResultsPage, RecipePage, HomePage
import urllib
from weboob.tools.compat import urlencode
__all__ = ['AllrecipesBrowser']
......@@ -47,7 +47,7 @@ class AllrecipesBrowser(PagesBrowser):
if not self.TOKEN:
self.fill_token()
return self.results.go(query=urllib.urlencode(query)).iter_recipes()
return self.results.go(query=urlencode(query)).iter_recipes()
def get_recipe(self, _id, obj=None):
if not self.TOKEN:
......
......@@ -19,10 +19,10 @@
from weboob.capabilities.recipe import CapRecipe, Recipe
from weboob.tools.backend import Module
from weboob.tools.compat import quote_plus
from .browser import AllrecipesBrowser
from urllib import quote_plus
__all__ = ['AllrecipesModule']
......
......@@ -21,11 +21,11 @@
from weboob.capabilities.bank import AccountNotFound
from weboob.browser import LoginBrowser, URL, need_login
from weboob.exceptions import BrowserIncorrectPassword
from weboob.tools.compat import unquote
import json
import os
from tempfile import mkstemp
from subprocess import check_output, STDOUT, CalledProcessError
from urllib import unquote
from .pages import SomePage, StatementsPage, StatementPage, SummaryPage, \
ActivityPage
......
......@@ -20,7 +20,6 @@
from datetime import datetime
import re
import urllib
from decimal import Decimal
from weboob.browser.pages import HTMLPage, RawPage, LoggedPage
......@@ -118,7 +117,7 @@ class LastPaymentsPage(LoggedPage, AmeliBasePage):
yield bil
def get_document(self, bill):
self.location(bill.url, urllib.urlencode(bill._args))
self.location(bill.url, params=bill._args)
class PaymentDetailsPage(AmeliBasePage):
......
......@@ -17,10 +17,10 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import urllib
from weboob.browser import LoginBrowser, URL, need_login
from weboob.exceptions import BrowserIncorrectPassword
from weboob.capabilities.bill import Detail
from weboob.tools.compat import urlencode
from decimal import Decimal
from .pages import LoginPage, HomePage, AccountPage, HistoryPage, BillsPage, SearchPage
......@@ -81,7 +81,7 @@ class AmeliProBrowser(LoginBrowser):
}
self.session.headers.update({'Content-Type': 'application/x-www-form-urlencoded'})
self.historyp.go(data=urllib.urlencode(data))
self.historyp.go(data=urlencode(data))
if self.historyp.is_here():
return self.page.iter_history()
......
......@@ -18,7 +18,6 @@
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from datetime import timedelta
import urllib
from weboob.capabilities.image import Thumbnail
from weboob.capabilities.base import BaseObject, NotAvailable
......@@ -29,8 +28,7 @@ from weboob.browser.elements import DictElement, ItemElement, ListElement, metho
from weboob.browser.filters.standard import Date, Format, Env, CleanText, Field, Regexp, Join, Eval
from weboob.browser.filters.json import Dict
from weboob.browser.filters.html import XPath
from weboob.tools.compat import basestring
from weboob.tools.compat import basestring, unquote
from .video import ArteVideo, ArteSiteVideo, SITE
......@@ -193,7 +191,7 @@ class VideosListPage(HTMLPage):
return self.doc.xpath('//div[@class="video-container"]')[0].attrib['arte_vp_url']
elif self.doc.xpath('//iframe'):
url = Regexp(CleanText('./@src'), '.*json_url=(.*)', default='')(self.doc.xpath('//iframe')[0])
return urllib.unquote(url)
return unquote(url)
return ''
......
......@@ -19,10 +19,10 @@
from weboob.capabilities.subtitle import CapSubtitle, LanguageNotSupported
from weboob.tools.backend import Module
from weboob.tools.compat import quote_plus
from .browser import AttilasubBrowser
from urllib import quote_plus
__all__ = ['AttilasubModule']
......
......@@ -23,7 +23,6 @@ from hashlib import sha256
from datetime import datetime
import math
import re
import urllib
import urllib2
from weboob.exceptions import BrowserIncorrectPassword, BrowserHTTPNotFound, BrowserUnavailable
......@@ -31,6 +30,7 @@ from weboob.deprecated.browser import Browser
from weboob.browser.browsers import LoginBrowser
from weboob.browser.pages import HTMLPage
from weboob.browser.filters.standard import CleanText
from weboob.tools.compat import urlencode
from weboob.tools.json import json
from weboob.tools.date import local2utc
from weboob.tools.misc import to_unicode
......@@ -81,7 +81,7 @@ class WebsiteBrowser(LoginBrowser):
'remember': 'on',
}
self.open('/auth/login', data=data)
#self.readurl('https://www.adopteunmec.com/auth/login', urllib.urlencode(data))
#self.readurl('https://www.adopteunmec.com/auth/login', urlencode(data))
def get_profile(self, id):
profile = {}
......@@ -179,7 +179,7 @@ class AuMBrowser(Browser):
# Always do POST requests.
data = ''
elif isinstance(data, (list,tuple,dict)):
data = urllib.urlencode(data)
data = urlencode(data)
elif isinstance(data, unicode):
data = data.encode('utf-8')
......
......@@ -19,11 +19,11 @@
from weboob.browser import PagesBrowser, URL
from weboob.tools.compat import urlencode
from .pages import DeparturesPage
from datetime import datetime
import urllib
class BlablacarBrowser(PagesBrowser):
......@@ -46,4 +46,4 @@ class BlablacarBrowser(PagesBrowser):
query['hb'] = _heure
query['he'] = '24'
return self.departures.open(qry=urllib.urlencode(query)).get_station_departures()
return self.departures.open(qry=urlencode(query)).get_station_departures()
......@@ -19,11 +19,11 @@
from datetime import datetime, timedelta
from weboob.tools.compat import urlsplit, parse_qsl
from weboob.browser import LoginBrowser, URL, need_login
from weboob.browser.browsers import StatesMixin
from weboob.exceptions import BrowserIncorrectPassword, BrowserBanned, NoAccountsException, BrowserUnavailable
from weboob.tools.compat import urlsplit, parse_qsl
from .pages import (
LoginPage, Initident, CheckPassword, repositionnerCheminCourant, BadLoginPage, AccountDesactivate,
......
......@@ -19,10 +19,10 @@
from weboob.capabilities.torrent import CapTorrent, Torrent
from weboob.tools.backend import Module
from weboob.tools.compat import quote_plus
from .browser import BtmonBrowser
from urllib import quote_plus
__all__ = ['BtmonModule']
......
......@@ -17,9 +17,9 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import urllib
from weboob.browser import PagesBrowser, URL
from weboob.tools.compat import quote_plus
from .pages import ChannelsPage, VideoPage
from .video import CanalplusVideo
......@@ -51,7 +51,7 @@ class CanalplusBrowser(PagesBrowser):
self.location('http://service.canal-plus.com/video/rest/initPlayer/cplus/')
def search_videos(self, pattern):
self.location('http://service.canal-plus.com/video/rest/search/cplus/' + urllib.quote_plus(pattern.replace('/', '').encode('utf-8')))
self.location('http://service.canal-plus.com/video/rest/search/cplus/' + quote_plus(pattern.replace('/', '').encode('utf-8')))
return self.page.iter_results()
def get_video(self, url, video=None):
......
......@@ -9,7 +9,6 @@
# http://sam.zoy.org/wtfpl/COPYING for more details.
from collections import OrderedDict
import urllib
import datetime
from weboob.capabilities.base import NotAvailable
......@@ -20,6 +19,7 @@ from weboob.deprecated.browser import Browser
from weboob.deprecated.browser.decorators import id2url
from weboob.capabilities.image import Thumbnail
from weboob.capabilities.video import BaseVideo
from weboob.tools.compat import quote_plus
__all__ = ['CappedBrowser']
......@@ -124,7 +124,7 @@ class CappedBrowser(Browser):
return self.page.get_video(video)
def search_videos(self, pattern):
self.location('/search?s=%s' % (urllib.quote_plus(pattern.encode('utf-8'))))
self.location('/search?s=%s' % (quote_plus(pattern.encode('utf-8'))))
assert self.is_on_page(IndexPage)
return self.page.iter_videos()
......
......@@ -19,10 +19,10 @@
from weboob.capabilities.torrent import CapTorrent, Torrent
from weboob.tools.backend import Module
from weboob.tools.compat import quote_plus
from .browser import CpasbienBrowser
from urllib import quote_plus
__all__ = ['CpasbienModule']
......
......@@ -21,7 +21,6 @@
import re
import ast
from urllib import quote
from decimal import Decimal
from io import BytesIO
from datetime import date as da
......@@ -36,6 +35,7 @@ from weboob.capabilities.profile import Profile
from weboob.capabilities import NotAvailable
from weboob.tools.capabilities.bank.transactions import FrenchTransaction
from weboob.tools.captcha.virtkeyboard import GridVirtKeyboard
from weboob.tools.compat import quote
from weboob.tools.json import json
......
......@@ -18,10 +18,6 @@
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
try:
from urlparse import urlparse
except ImportError:
from urllib.parse import urlparse
from datetime import datetime
from dateutil.relativedelta import relativedelta
......@@ -37,6 +33,7 @@ from weboob.browser.pages import FormNotFound
from weboob.exceptions import BrowserIncorrectPassword
from weboob.capabilities.bank import Account, AddRecipientStep, AddRecipientError, Recipient
from weboob.capabilities import NotAvailable
from weboob.tools.compat import urlparse
from .pages import LoginPage, LoginErrorPage, AccountsPage, UserSpacePage, \
OperationsPage, CardPage, ComingPage, NoOperationsPage, \
......
......@@ -21,10 +21,6 @@
import re
import hashlib
try:
from urlparse import urlparse, parse_qs
except ImportError:
from urllib.parse import urlparse, parse_qs
from decimal import Decimal, InvalidOperation
from dateutil.relativedelta import relativedelta
......@@ -42,6 +38,7 @@ from weboob.capabilities.contact import Advisor
from weboob.capabilities.profile import Profile
from weboob.tools.capabilities.bank.iban import is_iban_valid
from weboob.tools.capabilities.bank.transactions import FrenchTransaction
from weboob.tools.compat import urlparse, parse_qs
from weboob.tools.date import parse_french_date
from weboob.tools.value import Value
......
......@@ -17,10 +17,10 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from urllib import quote_plus
import re
from weboob.browser import PagesBrowser, URL
from weboob.tools.compat import quote_plus
from .pages import IndexPage, VideoPage
......
......@@ -18,10 +18,10 @@
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import urllib
from weboob.capabilities.translate import CapTranslate, Translation, TranslationFail, LanguageNotSupported
from weboob.tools.backend import Module
from weboob.tools.compat import urlencode
from weboob.deprecated.browser import StandardBrowser
......@@ -43,7 +43,7 @@ class EbonicsModule(Module, CapTranslate):
with self.browser:
data = {'English': text.encode('utf-8')}
doc = self.browser.location('http://joel.net/EBONICS/Translator', urllib.urlencode(data))
doc = self.browser.location('http://joel.net/EBONICS/Translator', urlencode(data))
try:
text = doc.getroot().cssselect('div.translateform div.bubble1 div.bubblemid')[0].text
except IndexError:
......
......@@ -18,7 +18,7 @@
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.deprecated.browser import Browser, BrowserIncorrectPassword
from urllib import urlencode
from weboob.tools.compat import urlencode
from .pages import IndexPage, GalleryPage, ImagePage, HomePage, LoginPage
from .gallery import EHentaiImage
......
......@@ -17,9 +17,9 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import urllib
from weboob.browser import PagesBrowser, URL
from weboob.capabilities.housing import Query, TypeNotSupported
from weboob.tools.compat import urlencode
from .pages import CitiesPage, SearchPage, HousingPage, HousingPage2, PhonePage
......@@ -67,7 +67,7 @@ class ExplorimmoBrowser(PagesBrowser):
'roomMin': nb_rooms or '',
'page': '1'}
query = u'%s%s%s' % (urllib.urlencode(data), '&type=', '&type='.join(ret))
query = u'%s%s%s' % (urlencode(data), '&type=', '&type='.join(ret))
return self.search.go(query=query).iter_housings()
......
......@@ -18,7 +18,6 @@
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import re
import urllib
from decimal import Decimal
from datetime import datetime
from weboob.browser.filters.json import Dict
......@@ -29,7 +28,7 @@ from weboob.browser.filters.html import Attr, CleanHTML, Link, XPath
from weboob.capabilities.base import NotAvailable, NotLoaded
from weboob.capabilities.housing import Housing, HousingPhoto, City, UTILITIES
from weboob.tools.capabilities.housing.housing import PricePerMeterFilter
from weboob.tools.compat import unquote
class CitiesPage(JsonPage):
......@@ -126,7 +125,7 @@ class SearchPage(HTMLPage):
default=None
)(self)
if url:
url = urllib.unquote(url)
url = unquote(url)
if "http://" in url[3:]:
url = url[url.find("http://", 3):url.rfind("?")]
return [HousingPhoto(url)]
......
......@@ -17,11 +17,11 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from urlparse import urlparse, parse_qs
from weboob.browser import LoginBrowser, URL
from weboob.browser.pages import HTMLPage, LoggedPage
from weboob.exceptions import BrowserIncorrectPassword
from weboob.tools.compat import urlparse, parse_qs
class GoogleLoginPage(LoggedPage, HTMLPage):
......
......@@ -20,7 +20,6 @@
from __future__ import unicode_literals
import datetime
import urlparse
from weboob.browser.pages import JsonPage, HTMLPage, pagination
from weboob.browser.filters.standard import CleanDecimal, CleanText, Date, Env, Format, Regexp
......@@ -29,6 +28,7 @@ from weboob.browser.elements import ItemElement, ListElement, method
from weboob.capabilities.base import NotAvailable
from weboob.capabilities.housing import City, Housing, UTILITIES
from weboob.tools.capabilities.housing.housing import PricePerMeterFilter
from weboob.tools.compat import urljoin
from .constants import AVAILABLE_TYPES, QUERY_TYPES, QUERY_HOUSE_TYPES
......@@ -226,7 +226,7 @@ class SearchResultsPage(HTMLPage):
)
def obj_url(self):
return urlparse.urljoin(
return urljoin(
self.page.browser.BASEURL,
Link('.//h3[has-class("TeaserOffer-title")]/a')(self)
)
......
......@@ -19,10 +19,9 @@
import re
import urlparse
from logging import warning, debug
from urlparse import parse_qs
from weboob.tools.compat import parse_qs, urlparse
from weboob.tools.misc import get_bytes_size
from weboob.tools.html import html2text
from weboob.capabilities.torrent import Torrent
......@@ -83,7 +82,7 @@ class TorrentsPage(BasePage):
title += u' (%s)' % tds[i].find('a').text
else:
title = ' - '.join([a.text for a in tds[i].findall('a')])
url = urlparse.urlparse(tds[i].find('a').attrib['href'])
url = urlparse(tds[i].find('a').attrib['href'])
params = parse_qs(url.query)
if 'torrentid' in params:
id = '%s.%s' % (params['id'][0], params['torrentid'][0])
......
......@@ -18,11 +18,11 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import urllib
from weboob.deprecated.browser import Browser, BrowserIncorrectPassword, BrowserUnavailable,\
BrowserBanned
from weboob.deprecated.browser.decorators import id2url
from weboob.tools.compat import urlencode
#from .pages.index import IndexPage
from .pages import VideoPage, IndexPage, SearchPage
......@@ -67,7 +67,7 @@ class GDCVaultBrowser(Browser):
}
data = self.readurl('http://gdcvault.com/api/login.php',
urllib.urlencode(params))
urlencode(params))
# some data returned as JSON, not sure yet if it's useful
if data is None:
......@@ -131,7 +131,7 @@ class GDCVaultBrowser(Browser):
"category" : "free",
"keyword" : pattern.encode('utf-8'),
"conference_id" : "", }
post_data = urllib.urlencode(post_data)
post_data = urlencode(post_data)
# probably not required
self.addheaders = [('Referer', 'http://gdcvault.com/'),
("Content-Type" , 'application/x-www-form-urlencoded') ]
......
......@@ -20,8 +20,8 @@
from weboob.deprecated.browser import Page
from weboob.tools.compat import quote
import urllib
import re
import datetime
from dateutil.parser import parse as parse_dt
......@@ -226,7 +226,7 @@ class VideoPage(Page):
# token doesn't actually seem required
# 1441 has a space in the xml filename
xml_filename = urllib.quote(m.group(2))
xml_filename = quote(m.group(2))
config_url = m.group(1) + xml_filename + '?token=' + m.group(3)
# self.browser.addheaders += [['Referer', 'http://gdcvault.com/play/%s' % self.group_dict['id']]]
......@@ -270,11 +270,11 @@ class VideoPage(Page):
obj = self.parser.select(config.getroot(), 'speakervideo', 1)
if obj.text is not None:
videos['speaker'] = 'rtmp://' + host + '/' + urllib.quote(obj.text)
videos['speaker'] = 'rtmp://' + host + '/' + quote(obj.text)
obj = self.parser.select(config.getroot(), 'slidevideo', 1)
if obj.text is not None:
videos['slides'] = 'rtmp://' + host + '/' + urllib.quote(obj.text)
videos['slides'] = 'rtmp://' + host + '/' + quote(obj.text)
# print videos
# XXX
......
......@@ -21,11 +21,11 @@
import datetime
import re
import os
from urllib import quote_plus
from weboob.browser.browsers import APIBrowser
from weboob.browser.cache import CacheMixin
from weboob.browser.exceptions import ClientError
from weboob.tools.compat import quote_plus
__all__ = ['GithubBrowser']
......
......@@ -19,10 +19,10 @@
from weboob.capabilities.cinema import CapCinema, Person, Movie
from weboob.tools.backend import Module
from weboob.tools.compat import quote_plus
from .browser import ImdbBrowser
from urllib import quote_plus
__all__ = ['ImdbModule']
......
......@@ -17,10 +17,9 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import urllib
from weboob.deprecated.browser import Browser
from weboob.deprecated.browser.decorators import id2url
from weboob.tools.compat import quote
from .video import JacquieEtMichelVideo
from .pages import VideoPage, ResultsPage
......@@ -44,7 +43,7 @@ class JacquieEtMichelBrowser(Browser):
return self.page.get_video(video)
def search_videos(self, pattern):
self.location('/videolist/searchmodevideo/query%s/' % (urllib.quote(pattern.encode('utf-8'))))
self.location('/videolist/searchmodevideo/query%s/' % (quote(pattern.encode('utf-8'))))
assert self.is_on_page(ResultsPage)
return self.page.iter_videos()
......
......@@ -19,10 +19,10 @@
from weboob.capabilities.torrent import CapTorrent, Torrent
from weboob.tools.backend import Module
from weboob.tools.compat import quote_plus
from .browser import KickassBrowser
from urllib import quote_plus
#from contextlib import closing
#from gzip import GzipFile
......
......@@ -19,11 +19,10 @@
from weboob.browser import PagesBrowser, URL
from weboob.tools.compat import quote_plus
from .pages import SearchPage, AdvertPage
import urllib
class LinuxJobsBrowser(PagesBrowser):
BASEURL = 'https://www.linuxjobs.fr'
......@@ -40,7 +39,7 @@ class LinuxJobsBrowser(PagesBrowser):
def search_job(self, pattern=None):
if pattern is None:
return []
self.search_page.go(job=urllib.quote_plus(pattern.encode('utf-8')))
self.search_page.go(job=quote_plus(pattern.encode('utf-8')))
assert self.search_page.is_here()
return self.page.iter_job_adverts()
......@@ -19,10 +19,10 @@
from weboob.capabilities.lyrics import CapLyrics, SongLyrics
from weboob.tools.backend import Module
from weboob.tools.compat import quote_plus
from .browser import LyricsdotcomBrowser
from urllib import quote_plus
__all__ = ['LyricsdotcomModule']
......
......@@ -19,10 +19,10 @@
from weboob.capabilities.lyrics import CapLyrics, SongLyrics
from weboob.tools.backend import Module
from weboob.tools.compat import quote_plus
from .browser import LyricsmodeBrowser
from urllib import quote_plus
__all__ = ['LyricsmodeModule']
......
......@@ -19,10 +19,10 @@
from weboob.capabilities.recipe import CapRecipe, Recipe
from weboob.tools.backend import Module
from weboob.tools.compat import quote_plus
from .browser import MarmitonBrowser
from urllib import quote_plus
__all__ = ['MarmitonModule']
......