diff --git a/modules/imdb/browser.py b/modules/imdb/browser.py
index 56fdc498896906c7905591c3a861a6507ef98715..786be53bdee83301158318c463d692c09f30f487 100644
--- a/modules/imdb/browser.py
+++ b/modules/imdb/browser.py
@@ -17,13 +17,20 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
-import re
+from __future__ import unicode_literals
-from HTMLParser import HTMLParser
-from weboob.deprecated.browser import Browser, BrowserHTTPNotFound
+import re
+try:
+ from HTMLParser import HTMLParser
+except ImportError:
+ from html.parser import HTMLParser
+
+from weboob.browser import PagesBrowser, URL
+from weboob.browser.profiles import Wget
+from weboob.exceptions import BrowserHTTPNotFound
from weboob.capabilities.base import NotAvailable, NotLoaded
from weboob.capabilities.cinema import Movie, Person
-from weboob.tools.json import json
+from weboob.tools.compat import unicode
from .pages import PersonPage, MovieCrewPage, BiographyPage, ReleasePage
@@ -32,21 +39,18 @@
__all__ = ['ImdbBrowser']
-class ImdbBrowser(Browser):
- DOMAIN = 'www.imdb.com'
- PROTOCOL = 'http'
- ENCODING = 'utf-8'
- USER_AGENT = Browser.USER_AGENTS['wget']
- PAGES = {
- 'http://www.imdb.com/title/tt[0-9]*/fullcredits.*': MovieCrewPage,
- 'http://www.imdb.com/title/tt[0-9]*/releaseinfo.*': ReleasePage,
- 'http://www.imdb.com/name/nm[0-9]*/*': PersonPage,
- 'http://www.imdb.com/name/nm[0-9]*/bio.*': BiographyPage,
- }
+class ImdbBrowser(PagesBrowser):
+ BASEURL = 'http://www.imdb.com'
+ PROFILE = Wget()
+
+ movie_crew = URL(r'/title/tt[0-9]*/fullcredits.*', MovieCrewPage)
+ release = URL(r'/title/tt[0-9]*/releaseinfo.*', ReleasePage)
+ bio = URL(r'/name/nm[0-9]*/bio.*', BiographyPage)
+ person = URL(r'/name/nm[0-9]*/*', PersonPage)
def iter_movies(self, pattern):
- res = self.readurl('http://www.imdb.com/xml/find?json=1&nr=1&tt=on&q=%s' % pattern.encode('utf-8'))
- jres = json.loads(res)
+ res = self.open('http://www.imdb.com/xml/find?json=1&nr=1&tt=on', params={'q': pattern})
+ jres = res.json()
htmlparser = HTMLParser()
for cat in ['title_popular', 'title_exact', 'title_approx']:
if cat in jres:
@@ -71,8 +75,8 @@ def iter_movies(self, pattern):
yield movie
def iter_persons(self, pattern):
- res = self.readurl('http://www.imdb.com/xml/find?json=1&nr=1&nm=on&q=%s' % pattern.encode('utf-8'))
- jres = json.loads(res)
+ res = self.open('http://www.imdb.com/xml/find?json=1&nr=1&nm=on', params={'q': pattern})
+ jres = res.json()
htmlparser = HTMLParser()
for cat in ['name_popular', 'name_exact', 'name_approx']:
if cat in jres:
@@ -91,9 +95,9 @@ def iter_persons(self, pattern):
yield person
def get_movie(self, id):
- res = self.readurl('http://www.omdbapi.com/?i=%s&plot=full' % id)
+ res = self.open('http://www.omdbapi.com/?apikey=b7c56eb5&i=%s&plot=full' % id)
if res is not None:
- jres = json.loads(res)
+ jres = res.json()
else:
return None
htmlparser = HTMLParser()
@@ -178,38 +182,38 @@ def get_person(self, id):
self.location('http://www.imdb.com/name/%s' % id)
except BrowserHTTPNotFound:
return
- assert self.is_on_page(PersonPage)
+ assert self.person.is_here()
return self.page.get_person(id)
def get_person_biography(self, id):
self.location('http://www.imdb.com/name/%s/bio' % id)
- assert self.is_on_page(BiographyPage)
+ assert self.bio.is_here()
return self.page.get_biography()
def iter_movie_persons(self, movie_id, role):
self.location('http://www.imdb.com/title/%s/fullcredits' % movie_id)
- assert self.is_on_page(MovieCrewPage)
+ assert self.movie_crew.is_here()
for p in self.page.iter_persons(role):
yield p
def iter_person_movies(self, person_id, role):
self.location('http://www.imdb.com/name/%s' % person_id)
- assert self.is_on_page(PersonPage)
+ assert self.person.is_here()
return self.page.iter_movies(role)
def iter_person_movies_ids(self, person_id):
self.location('http://www.imdb.com/name/%s' % person_id)
- assert self.is_on_page(PersonPage)
+ assert self.person.is_here()
for movie in self.page.iter_movies_ids():
yield movie
def iter_movie_persons_ids(self, movie_id):
self.location('http://www.imdb.com/title/%s/fullcredits' % movie_id)
- assert self.is_on_page(MovieCrewPage)
+ assert self.movie_crew.is_here()
for person in self.page.iter_persons_ids():
yield person
def get_movie_releases(self, id, country):
self.location('http://www.imdb.com/title/%s/releaseinfo' % id)
- assert self.is_on_page(ReleasePage)
+ assert self.release.is_here()
return self.page.get_movie_releases(country)
diff --git a/modules/imdb/module.py b/modules/imdb/module.py
index b368f1bd4649454fa38be8fab0112a6be2debfcf..aa7755de85f4b19c9c0e929dfd738b24971770d1 100644
--- a/modules/imdb/module.py
+++ b/modules/imdb/module.py
@@ -17,9 +17,10 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
+from __future__ import unicode_literals
+
from weboob.capabilities.cinema import CapCinema, Person, Movie
from weboob.tools.backend import Module
-from weboob.tools.compat import quote_plus
from .browser import ImdbBrowser
@@ -43,10 +44,10 @@ def get_person(self, id):
return self.browser.get_person(id)
def iter_movies(self, pattern):
- return self.browser.iter_movies(quote_plus(pattern.encode('utf-8')))
+ return self.browser.iter_movies(pattern)
def iter_persons(self, pattern):
- return self.browser.iter_persons(quote_plus(pattern.encode('utf-8')))
+ return self.browser.iter_persons(pattern)
def iter_movie_persons(self, id, role=None):
return self.browser.iter_movie_persons(id, role)
diff --git a/modules/imdb/pages.py b/modules/imdb/pages.py
index 157ee664f2032526749f4ca835ece877e0c7dbe7..5ef0a1a45c5b06afc5e55e7bd81025e754012e93 100644
--- a/modules/imdb/pages.py
+++ b/modules/imdb/pages.py
@@ -17,22 +17,24 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
+from __future__ import unicode_literals
-from weboob.capabilities.cinema import Person, Movie
-from weboob.capabilities.base import NotAvailable, NotLoaded
-from weboob.deprecated.browser import Page
-from weboob.tools.html import html2text
from datetime import datetime
import re
+from weboob.capabilities.cinema import Person, Movie
+from weboob.capabilities.base import NotAvailable, NotLoaded
+from weboob.browser.pages import HTMLPage
+from weboob.browser.filters.html import CleanHTML
+
-class ReleasePage(Page):
+class ReleasePage(HTMLPage):
''' Page containing releases of a movie
'''
def get_movie_releases(self, country_filter):
- result = unicode()
- links = self.parser.select(self.document.getroot(), 'table#release_dates a')
+ result = ''
+ links = self.doc.xpath('//table[@id="release_dates"]//a')
for a in links:
href = a.attrib.get('href', '')
@@ -40,13 +42,13 @@ def get_movie_releases(self, country_filter):
if href.strip('/').split('/')[0] == 'calendar' and\
(country_filter is None or re.search('region=([a-zA-Z]+)&', href).group(1).lower() == country_filter):
country = a.text
- td_date = self.parser.select(a.getparent().getparent().getparent(), 'td')[1]
- date_links = self.parser.select(td_date, 'a')
+ td_date = a.xpath('./../../..//td')[1]
+ date_links = td_date.xpath('.//a')
if len(date_links) > 1:
date = date_links[1].attrib.get('href', '').strip('/').split('/')[-1]
date += '-'+date_links[0].attrib.get('href', '').strip('/').split('/')[-1]
else:
- date = unicode(self.parser.select(a.getparent().getparent().getparent(), 'td')[1].text_content())
+ date = a.xpath('./../../..//td')[1].text_content()
result += '%s : %s\n' % (country, date)
if result == u'':
result = NotAvailable
@@ -55,39 +57,39 @@ def get_movie_releases(self, country_filter):
return result
-class BiographyPage(Page):
+class BiographyPage(HTMLPage):
''' Page containing biography of a person
'''
def get_biography(self):
- bio = unicode()
+ bio = ''
start = False
- tn = self.parser.select(self.document.getroot(), 'div#bio_content', 1)
- for el in tn.getchildren():
+ tn = self.doc.xpath('//div[@id="bio_content"]')[0]
+ for el in tn.xpath('./*'):
if el.attrib.get('name') == 'mini_bio':
start = True
if start:
- bio += html2text(self.parser.tostring(el))
+ bio += CleanHTML('.')(el)
return bio
-class MovieCrewPage(Page):
+class MovieCrewPage(HTMLPage):
''' Page listing all the persons related to a movie
'''
def iter_persons(self, role_filter=None):
if (role_filter is None or (role_filter is not None and role_filter == 'actor')):
- tables = self.parser.select(self.document.getroot(), 'table.cast_list')
+ tables = self.doc.xpath('//table[has-class("cast_list")]')
if len(tables) > 0:
table = tables[0]
- tds = self.parser.select(table, 'td.itemprop')
+ tds = table.xpath('.//td[has-class("itemprop")]')
for td in tds:
id = td.find('a').attrib.get('href', '').strip('/').split('/')[1]
- name = unicode(td.find('a').text)
- char_name = unicode(self.parser.select(td.getparent(), 'td.character', 1).text_content())
+ name = td.find('a').text
+ char_name = td.xpath('..//td[has-class("character")]')[0].text_content()
person = Person(id, name)
person.short_description = char_name
person.real_name = NotLoaded
@@ -101,35 +103,35 @@ def iter_persons(self, role_filter=None):
person.thumbnail_url = NotLoaded
yield person
- for gloss_link in self.parser.select(self.document.getroot(), 'table[cellspacing="1"] h5 a'):
+ for gloss_link in self.doc.xpath('//table[@cellspacing="1"]//h5//a'):
role = gloss_link.attrib.get('name', '').rstrip('s')
if (role_filter is None or (role_filter is not None and role == role_filter)):
tbody = gloss_link.getparent().getparent().getparent().getparent()
- for line in self.parser.select(tbody, 'tr')[1:]:
- for a in self.parser.select(line, 'a'):
+ for line in tbody.xpath('.//tr')[1:]:
+ for a in line.xpath('.//a'):
role_detail = NotAvailable
href = a.attrib.get('href', '')
if '/name/nm' in href:
id = href.strip('/').split('/')[-1]
- name = unicode(a.text)
+ name = a.text
if 'glossary' in href:
- role_detail = unicode(a.text)
+ role_detail = a.text
person = Person(id, name)
person.short_description = role_detail
yield person
# yield self.browser.get_person(id)
def iter_persons_ids(self):
- tables = self.parser.select(self.document.getroot(), 'table.cast_list')
+ tables = self.doc.xpath('//table[has-class("cast_list")]')
if len(tables) > 0:
table = tables[0]
- tds = self.parser.select(table, 'td.itemprop')
+ tds = table.xpath('.//td[has-class("itemprop")]')
for td in tds:
id = td.find('a').attrib.get('href', '').strip('/').split('/')[1]
yield id
-class PersonPage(Page):
+class PersonPage(HTMLPage):
''' Page informing about a person
It is used to build a Person instance and to get the movie list related to a person
'''
@@ -146,35 +148,35 @@ def get_person(self, id):
thumbnail_url = NotAvailable
roles = {}
nationality = NotAvailable
- td_overview = self.parser.select(self.document.getroot(), 'td#overview-top', 1)
- descs = self.parser.select(td_overview, 'span[itemprop=description]')
+ td_overview = self.doc.xpath('//td[@id="overview-top"]')[0]
+ descs = td_overview.xpath('.//span[@itemprop="description"]')
if len(descs) > 0:
- short_biography = unicode(descs[0].text)
- rname_block = self.parser.select(td_overview, 'div.txt-block h4.inline')
+ short_biography = descs[0].text
+ rname_block = td_overview.xpath('.//div[has-class("txt-block")]//h4[has-class("inline")]')
if len(rname_block) > 0 and "born" in rname_block[0].text.lower():
- links = self.parser.select(rname_block[0].getparent(), 'a')
+ links = rname_block[0].xpath('..//a')
for a in links:
href = a.attrib.get('href', '').strip()
if href == 'bio':
- real_name = unicode(a.text.strip())
+ real_name = a.text.strip()
elif 'birth_place' in href:
- birth_place = unicode(a.text.lower().strip())
- names = self.parser.select(td_overview, 'h1 span[itemprop=name]')
+ birth_place = a.text.lower().strip()
+ names = td_overview.xpath('.//h1//span[@itemprop="name"]')
if len(names) > 0:
- name = unicode(names[0].text.strip())
- times = self.parser.select(td_overview, 'time[itemprop=birthDate]')
+ name = names[0].text.strip()
+ times = td_overview.xpath('.//time[@itemprop="birthDate"]')
if len(times) > 0:
time = times[0].attrib.get('datetime', '').split('-')
if len(time) == 3 and int(time[0]) >= 1900:
birth_date = datetime(int(time[0]), int(time[1]), int(time[2]))
- dtimes = self.parser.select(td_overview, 'time[itemprop=deathDate]')
+ dtimes = td_overview.xpath('.//time[@itemprop="deathDate"]')
if len(dtimes) > 0:
dtime = dtimes[0].attrib.get('datetime', '').split('-')
if len(dtime) == 3 and int(dtime[0]) >= 1900:
death_date = datetime(int(dtime[0]), int(dtime[1]), int(dtime[2]))
- img_thumbnail = self.parser.select(self.document.getroot(), 'td#img_primary img')
+ img_thumbnail = self.doc.xpath('//td[@id="img_primary img"]')
if len(img_thumbnail) > 0:
- thumbnail_url = unicode(img_thumbnail[0].attrib.get('src', ''))
+ thumbnail_url = img_thumbnail[0].attrib.get('src', '')
roles = self.get_roles()
@@ -192,26 +194,26 @@ def get_person(self, id):
return person
def iter_movies_ids(self):
- for role_div in self.parser.select(self.document.getroot(), 'div#filmography div.filmo-category-section > div'):
- for a in self.parser.select(role_div, 'a'):
+ for role_div in self.doc.xpath('//div[@id="filmography"]//div[has-class("filmo-category-section")]/div'):
+ for a in role_div.xpath('.//a'):
m = re.search('/title/(tt.*)/\?.*', a.attrib.get('href'))
if m:
yield m.group(1)
def get_roles(self):
roles = {}
- for role_div in self.parser.select(self.document.getroot(), 'div#filmography > div.head'):
- role = self.parser.select(role_div, 'a')[-1].text
+ for role_div in self.doc.xpath('//div[@id="filmography"]/div[has-class("head")]'):
+ role = role_div.xpath('.//a')[-1].text
roles[role] = []
category = role_div.attrib.get('data-category')
- for infos in self.parser.select(self.document.getroot(), 'div#filmography > div.filmo-category-section > div'):
+ for infos in self.doc.xpath('//div[@id="filmography"]/div[has-class("filmo-category-section")]/div'):
if category in infos.attrib.get('id'):
roles[role].append(('N/A',infos.text_content().replace('\n', ' ').strip()))
return roles
def iter_movies(self, role_filter=None):
- for role_div in self.parser.select(self.document.getroot(), 'div#filmography > div.filmo-category-section > div'):
- for a in self.parser.select(role_div, 'a'):
+ for role_div in self.doc.xpath('//div[@id="filmography"]/div[has-class("filmo-category-section")]/div'):
+ for a in role_div.xpath('.//a'):
m = re.search('/title/(tt.*)/\?.*', a.attrib.get('href'))
if m:
yield Movie(m.group(1), a.text)
diff --git a/tools/py3-compatible.modules b/tools/py3-compatible.modules
index 23a235eb868ae36e26c84431fa547165e7ce2653..4ecdfc7a7548e279d6dd0a7f3924d189bf1accf9 100644
--- a/tools/py3-compatible.modules
+++ b/tools/py3-compatible.modules
@@ -39,6 +39,7 @@ funmooc/
groupamaes/
hsbc/
hybride/
+imdb/
indeed/
infomaniak/
ing/