From 28b2766942e220cc61218d49202376be88e55d7d Mon Sep 17 00:00:00 2001 From: Julien Veyssier Date: Wed, 28 Mar 2018 17:57:37 +0200 Subject: [PATCH] [limetorrents] add new captorrent module --- modules/limetorrents/__init__.py | 22 +++++++++ modules/limetorrents/browser.py | 49 +++++++++++++++++++ modules/limetorrents/favicon.png | Bin 0 -> 1283 bytes modules/limetorrents/module.py | 62 +++++++++++++++++++++++ modules/limetorrents/pages.py | 81 +++++++++++++++++++++++++++++++ modules/limetorrents/test.py | 40 +++++++++++++++ tools/py3-compatible.modules | 1 + 7 files changed, 255 insertions(+) create mode 100644 modules/limetorrents/__init__.py create mode 100644 modules/limetorrents/browser.py create mode 100644 modules/limetorrents/favicon.png create mode 100644 modules/limetorrents/module.py create mode 100644 modules/limetorrents/pages.py create mode 100644 modules/limetorrents/test.py diff --git a/modules/limetorrents/__init__.py b/modules/limetorrents/__init__.py new file mode 100644 index 0000000000..802f87827b --- /dev/null +++ b/modules/limetorrents/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from .module import LimetorrentsModule + +__all__ = ['LimetorrentsModule'] diff --git a/modules/limetorrents/browser.py b/modules/limetorrents/browser.py new file mode 100644 index 0000000000..6dff57411c --- /dev/null +++ b/modules/limetorrents/browser.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.browser.exceptions import BrowserHTTPNotFound +from weboob.browser import PagesBrowser +from weboob.browser.url import URL +from weboob.browser.profiles import Wget + +from .pages import SearchPage, TorrentPage + + +__all__ = ['LimetorrentsBrowser'] + + +class LimetorrentsBrowser(PagesBrowser): + PROFILE = Wget() + TIMEOUT = 30 + + BASEURL = 'https://www.limetorrents.cc/' + search = URL(r'/search/all/(?P.*)/seeds/(?P[0-9]+)/', SearchPage) + torrent = URL(r'/(?P.*)-torrent-(?P[0-9]+)\.html', TorrentPage) + + def iter_torrents(self, pattern): + return self.search.go(pattern=pattern, page=1).iter_torrents() + + def get_torrent(self, id): + try: + self.torrent.go(torrent_id=id, torrent_name='whatever') + torrent = self.page.get_torrent() + return torrent + except BrowserHTTPNotFound: + return diff --git a/modules/limetorrents/favicon.png b/modules/limetorrents/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..dbc8e2c9c8bf30f12dd2afa68a781d30b4469395 GIT binary patch literal 1283 zcmV+e1^oJnP))X0ls zQxXzUUK&vRCnP?ZsC`gNAQ&M=+a=VPNCGKDN-n2-Msx2IE1XT*; zqj-s$!1`QwKrn4jTWB?wK9`i^x}FO6-Hh7es3~^o42ytaYY*rSAd2P#j>o>Axe+iC zDu*uu56#{Pn1M`f1hm`RQGA?~FFSL2n!!X(nX>}oTd^nsT?}dwFc0WM^AlR{M+dvG zV!KfKNwg{g`cRxk>vt%3Px)T+1O?>T4Jf@CrB(nAn)9GPr`vrmdHrc|KqgdtJ&N5T zczX&^JOS!)RLn(TgBlrfH?R>@Ep4GKR1CL_(fOnOAVJ*<6nS)pQIvryV_EQ7PP&g# zMPKfxfI=k}@7rgg;s?f@D_WUPP=5=mD1*`x!odT%Sp%ZZLi(IM;<|oJyVhS24K888 zS%l{3=vvfLHiU|k@`7MJs3o4;bOXIk0fW|Z&|iAyJ;wB()x0zacsT}H}-`xhpq%S~TpzPMyfXwlKC6UJf+&Ox# z2y_#Ud-v-uk8q#o_eLVNPODy9g`@t+#TuYJm0A%P{XI08a?Qrq{VZ7@NG%Vk}*Tt4{OiaOdP5V5x&d z?1TQs=r<{M_fD$={tGsf?`{VcNJ!LSQ%9kl%&Uam@f|?t?RONe9_S&aH=+3_Z}6Y2tkIigTY`h+>CGJ{Fv_ZXAawL1IK`F zpmcUT45b`+ftuvQ?-bB1yWZ~8_dIYO7y>Qd;rf4d7FeS4nPr8I<{7i@bCkc6WISc!xNxQVM5Q4)DHoR8#aw?cAiY=cnl#knfO> zZI+MIs(?>`KY$8gKTw~o@f}j6EZ}u9Zl{E?2HGazfU^=&NIKmiX>dQVObXU`7?Q|9 z?gn#!. + +from weboob.capabilities.torrent import CapTorrent, Torrent +from weboob.tools.backend import Module +from weboob.tools.compat import quote_plus + +from .browser import LimetorrentsBrowser + + +__all__ = ['LimetorrentsModule'] + + +class LimetorrentsModule(Module, CapTorrent): + NAME = 'limetorrents' + MAINTAINER = u'Julien Veyssier' + EMAIL = 'eneiluj@posteo.net' + VERSION = '1.4' + DESCRIPTION = 'Limetorrents BitTorrent tracker' + LICENSE = 'AGPLv3+' + BROWSER = LimetorrentsBrowser + + def get_torrent(self, id): + return self.browser.get_torrent(id) + + def get_torrent_file(self, id): + torrent = self.browser.get_torrent(id) + if not torrent: + return None + + resp = self.browser.open(torrent.url) + return resp.content + + def iter_torrents(self, pattern): + return self.browser.iter_torrents(quote_plus(pattern.encode('utf-8'))) + + def fill_torrent(self, torrent, fields): + if 'files' in fields: + tor = self.get_torrent(torrent.id) + torrent.magnet = tor.magnet + torrent.files = tor.files + return torrent + + OBJECTS = { + Torrent:fill_torrent + } diff --git a/modules/limetorrents/pages.py b/modules/limetorrents/pages.py new file mode 100644 index 0000000000..66ef9e73b5 --- /dev/null +++ b/modules/limetorrents/pages.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . +import re + +from weboob.tools.misc import get_bytes_size +from weboob.capabilities.torrent import Torrent +from weboob.capabilities.base import NotAvailable + +from weboob.browser.elements import ItemElement, ListElement, method +from weboob.browser.pages import HTMLPage, pagination +from weboob.browser.filters.standard import Regexp, CleanText, CleanDecimal, Format +from weboob.browser.filters.html import AbsoluteLink + + +class SearchPage(HTMLPage): + @pagination + @method + class iter_torrents(ListElement): + next_page = AbsoluteLink('//a[@id="next"]') + item_xpath = '//table[has-class("table2")]//tr[position()>1]' + + class item(ItemElement): + klass = Torrent + def obj_url(self): + url = AbsoluteLink('.//div[has-class("tt-name")]/a[1]')(self) + return url.replace('http://', 'https://') + obj_id = Regexp(CleanText('.//div[has-class("tt-name")]/a[2]/@href'), '/.*-torrent-([0-9]+)\.html$', '\\1') + obj_name = CleanText('.//div[has-class("tt-name")]/a[2]/text()') + obj_seeders = CleanDecimal('.//td[has-class("tdseed")]', default=0) + obj_leechers = CleanDecimal('.//td[has-class("tdleech")]', default=0) + obj_filename = Format('%s.torrent', obj_name) + + def obj_size(self): + rawsize = CleanText('(.//td[has-class("tdnormal")])[2]')(self) + nsize = float(re.sub(r'[A-Za-z]', '', rawsize)) + usize = re.sub(r'[.0-9 ]', '', rawsize).upper() + size = get_bytes_size(nsize, usize) + return size + + +class TorrentPage(HTMLPage): + @method + class get_torrent(ItemElement): + klass = Torrent + obj_name = CleanText('.//div[@id="content"]/h1') + obj_id = Regexp(CleanText('//div[@id="updatestatslink"]/a/@onclick'), 'torrent_id=([0-9]+)&', '\\1') + def obj_url(self): + url = AbsoluteLink('//div[has-class("torrentinfo")]//div[has-class("dltorrent")]//a[text()="Download torrent"]')(self) + return url.replace('http://', 'https://') + obj_filename = Format('%s.torrent', obj_name) + def obj_size(self): + s = CleanText('//td/b[text()="Size"]/../../td[2]')(self) + nsize = float(re.sub(r'[A-Za-z]', '', s)) + usize = re.sub(r'[.0-9 ]', '', s).upper() + size = get_bytes_size(nsize, usize) + return size + def obj_files(self): + res = [] + for f in self.xpath('//div[has-class("fileline")]'): + res.append(CleanText(f)(self)) + return res + obj_seeders = CleanDecimal('//div[@id="content"]/span[has-class("greenish")]', default=0) + obj_leechers = CleanDecimal('//div[@id="content"]/span[has-class("reddish")]', default=0) + obj_magnet = AbsoluteLink('//div[has-class("torrentinfo")]//div[has-class("dltorrent")]//a[text()="Magnet Link"]') + diff --git a/modules/limetorrents/test.py b/modules/limetorrents/test.py new file mode 100644 index 0000000000..af8ed86b63 --- /dev/null +++ b/modules/limetorrents/test.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.tools.test import BackendTest + +from random import choice + + +class LimetorrentsTest(BackendTest): + MODULE = 'limetorrents' + + def test_torrent(self): + torrents = list(self.backend.iter_torrents('spiderman')) + assert len(torrents) > 0 + for torrent in torrents: + assert torrent.id + assert torrent.name + assert torrent.url + + # get the file of a random torrent + # from the list (getting them all would be too long) + if len(torrents): + torrent = choice(torrents) + self.backend.get_torrent_file(torrent.id) diff --git a/tools/py3-compatible.modules b/tools/py3-compatible.modules index abf51adbc6..447c874490 100644 --- a/tools/py3-compatible.modules +++ b/tools/py3-compatible.modules @@ -58,6 +58,7 @@ lameteoagricole/ larousse/ lcl/ leboncoin/ +limetorrents/ linebourse/ logicimmo/ lolix/ -- GitLab