pax_global_header 0000666 0000000 0000000 00000000064 11666415431 0014521 g ustar 00root root 0000000 0000000 52 comment=037c35f40908f1833bda001fd8cdb2c9338a53d4
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/ 0000775 0000000 0000000 00000000000 11666415431 0020650 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/ 0000775 0000000 0000000 00000000000 11666415431 0022125 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/__init__.py 0000664 0000000 0000000 00000000001 11666415431 0024225 0 ustar 00root root 0000000 0000000
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/ 0000775 0000000 0000000 00000000000 11666415431 0024613 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/__init__.py 0000664 0000000 0000000 00000000001 11666415431 0026713 0 ustar 00root root 0000000 0000000
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/boobank/ 0000775 0000000 0000000 00000000000 11666415431 0026226 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/boobank/__init__.py 0000664 0000000 0000000 00000001426 11666415431 0030342 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .boobank import Boobank
__all__ = ['Boobank']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/boobank/boobank.py 0000664 0000000 0000000 00000021111 11666415431 0030207 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon, Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import sys
from weboob.capabilities.bank import ICapBank
from weboob.tools.application.repl import ReplApplication
from weboob.tools.application.formatters.iformatter import IFormatter
__all__ = ['Boobank']
class QifFormatter(IFormatter):
MANDATORY_FIELDS = ('id', 'date', 'label', 'amount', 'category')
count = 0
def flush(self):
self.count = 0
def format_dict(self, item):
result = u''
if self.count == 0:
result += u'!type:Bank\n'
result += u'D%s\n' % item['date'].strftime('%d/%m/%y')
result += u'T%s\n' % item['amount']
if item['category']:
result += u'N%s\n' % item['category']
result += u'M%s\n' % item['label']
result += u'^\n'
self.count += 1
return result
class TransferFormatter(IFormatter):
MANDATORY_FIELDS = ('id', 'date', 'origin', 'recipient', 'amount')
def flush(self):
pass
def format_dict(self, item):
result = u'------- Transfer %s -------\n' % item['id']
result += u'Date: %s\n' % item['date']
result += u'Origin: %s\n' % item['origin']
result += u'Recipient: %s\n' % item['recipient']
result += u'Amount: %.2f\n' % item['amount']
return result
class RecipientListFormatter(IFormatter):
MANDATORY_FIELDS = ('id', 'label')
count = 0
def flush(self):
self.count = 0
def format_dict(self, item):
self.count += 1
if self.interactive:
backend = item['id'].split('@', 1)[1]
id = '#%d (%s)' % (self.count, backend)
else:
id = item['id']
return u'%s %-30s %s %s' % (self.BOLD, id, self.NC, item['label'])
class AccountListFormatter(IFormatter):
MANDATORY_FIELDS = ('id', 'label', 'balance', 'coming')
count = 0
tot_balance = 0.0
tot_coming = 0.0
def flush(self):
if self.count < 1:
return
result = u'------------------------------------------%s+----------+----------\n' % (('-' * 15) if not self.interactive else '')
result +=u'%s Total %8s %8s' % ((' ' * 15) if not self.interactive else '',
'%.2f' % self.tot_balance, '%.2f' % self.tot_coming)
self.after_format(result)
self.tot_balance = 0.0
self.tot_coming = 0.0
self.count = 0
def format_dict(self, item):
self.count += 1
if self.interactive:
backend = item['id'].split('@', 1)[1]
id = '#%d (%s)' % (self.count, backend)
else:
id = item['id']
result = u''
if self.count == 1:
result += ' %s Account Balance Coming \n' % ((' ' * 15) if not self.interactive else '')
result += '------------------------------------------%s+----------+----------\n' % (('-' * 15) if not self.interactive else '')
result += (u' %s%-' + (u'15' if self.interactive else '30') + u's%s %-25s %8s %8s') % \
(self.BOLD, id, self.NC,
item['label'], '%.2f' % item['balance'], '%.2f' % (item['coming'] or 0.0))
self.tot_balance += item['balance']
if item['coming']:
self.tot_coming += item['coming']
return result
class Boobank(ReplApplication):
APPNAME = 'boobank'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon, Christophe Benz'
CAPS = ICapBank
DESCRIPTION = "Console application allowing to list your bank accounts and get their balance, " \
"display accounts history and coming bank operations, and transfer money from an account to " \
"another (if available)."
EXTRA_FORMATTERS = {'account_list': AccountListFormatter,
'recipient_list': RecipientListFormatter,
'transfer': TransferFormatter,
'qif': QifFormatter,
}
DEFAULT_FORMATTER = 'table'
COMMANDS_FORMATTERS = {'ls': 'account_list',
'list': 'account_list',
'transfer': 'transfer',
}
def _complete_account(self, exclude=None):
if exclude:
exclude = '%s@%s' % self.parse_id(exclude)
return [s for s in self._complete_object() if s != exclude]
def do_list(self, line):
"""
list
List accounts.
"""
return self.do_ls(line)
def complete_history(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_account()
def do_history(self, id):
"""
history ID
Display old operations.
"""
id, backend_name = self.parse_id(id)
if not id:
print >>sys.stderr, 'Error: please give an account ID (hint: use list command)'
return 2
names = (backend_name,) if backend_name is not None else None
def do(backend):
account = backend.get_account(id)
return backend.iter_history(account)
for backend, operation in self.do(do, backends=names):
self.format(operation)
self.flush()
def complete_coming(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_account()
def do_coming(self, id):
"""
coming ID
Display all future operations.
"""
id, backend_name = self.parse_id(id)
names = (backend_name,) if backend_name is not None else None
def do(backend):
account = backend.get_account(id)
return backend.iter_operations(account)
for backend, operation in self.do(do, backends=names):
self.format(operation)
self.flush()
def complete_transfer(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_account()
if len(args) == 3:
return self._complete_account(args[1])
def do_transfer(self, line):
"""
transfer ACCOUNT [RECIPIENT AMOUNT [REASON]]
Make a transfer beetwen two account
- ACCOUNT the source account
- RECIPIENT the recipient
- AMOUNT amount to transfer
- REASON reason of transfer
If you give only the ACCOUNT parameter, it lists all the
available recipients for this account.
"""
id_from, id_to, amount, reason = self.parse_command_args(line, 4, 1)
id_from, backend_name_from = self.parse_id(id_from)
if not id_to:
self.objects = []
self.set_formatter('recipient_list')
self.set_formatter_header(u'Available recipients')
names = (backend_name_from,) if backend_name_from is not None else None
for backend, recipient in self.do('iter_transfer_recipients', id_from, backends=names):
self.format(recipient)
self.add_object(recipient)
self.flush()
return 0
id_to, backend_name_to = self.parse_id(id_to)
try:
amount = float(amount)
except (TypeError,ValueError):
print >>sys.stderr, 'Error: please give a decimal amount to transfer'
return 2
if backend_name_from != backend_name_to:
print >>sys.stderr, "Transfer between different backends is not implemented"
return 4
else:
backend_name = backend_name_from
names = (backend_name,) if backend_name is not None else None
for backend, transfer in self.do('transfer', id_from, id_to, amount, reason, backends=names):
self.format(transfer)
self.flush()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/boobathon/ 0000775 0000000 0000000 00000000000 11666415431 0026566 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/boobathon/__init__.py 0000664 0000000 0000000 00000001426 11666415431 0030702 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .boobathon import Boobathon
__all__ = ['Boobathon']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/boobathon/boobathon.py 0000664 0000000 0000000 00000065335 11666415431 0031127 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import datetime, timedelta
import re
import sys
from urlparse import urlsplit
from random import choice
from weboob.capabilities.content import ICapContent
from weboob.tools.application.repl import ReplApplication
from weboob.tools.ordereddict import OrderedDict
__all__ = ['Boobathon']
class Task(object):
STATUS_NONE = 0
STATUS_PROGRESS = 1
STATUS_DONE = 2
def __init__(self, backend, capability):
self.backend = backend
self.capability = capability
self.status = self.STATUS_NONE
self.date = None
self.branch = u''
def __repr__(self):
return '' % (self.backend, self.capability)
class Member(object):
def __init__(self, id, name):
self.name = name
self.id = id
self.tasks = []
self.availabilities = u''
self.repository = None
self.hardware = u''
self.is_me = False
def shortname(self):
name = self.name
if len(name) > 20:
name = '%s..' % name[:18]
return name
class Event(object):
def __init__(self, name, backend):
self.my_id = backend.browser.get_userid()
self.name = 'wiki/weboob/%s' % name
self.description = None
self.date = None
self.begin = None
self.end = None
self.location = None
self.winner = None
self.backend = backend
self.members = OrderedDict()
self.load()
def get_me(self):
return self.members.get(self.backend.browser.get_userid(), None)
def currently_in_event(self):
if not self.date or not self.begin or not self.end:
return False
return self.begin < datetime.now() < self.end
def is_closed(self):
return self.end < datetime.now()
def format_duration(self):
if not self.begin or not self.end:
return None
delta = self.end - self.begin
return '%02d:%02d' % (delta.seconds/3600, delta.seconds%3600)
def check_time_coherence(self):
"""
Check if the end's day is before the begin's one, in
case it stops at the next day (eg. 15h->1h).
If it occures, add a day.
"""
if self.begin > self.end:
self.end = self.end + timedelta(1)
def load(self):
self.content = self.backend.get_content(self.name)
self.members.clear()
member = None
for line in self.content.content.split('\n'):
line = line.strip()
if line.startswith('h1. '):
self.title = line[4:]
elif line.startswith('h3=. '):
m = re.match('h3=. Event finished. Winner is "(.*)":/users/(\d+)\!', line)
if not m:
print >>sys.stderr, 'Unable to parse h3=: %s' % line
continue
self.winner = Member(int(m.group(2)), m.group(1))
elif line.startswith('h2. '):
continue
elif line.startswith('h3. '):
m = re.match('h3. "(.*)":/users/(\d+)', line)
if not m:
print >>sys.stderr, 'Unable to parse user "%s"' % line
continue
member = Member(int(m.group(2)), m.group(1))
if member.id == self.my_id:
member.is_me = True
if self.winner is not None and member.id == self.winner.id:
self.winner = member
self.members[member.id] = member
elif self.description is None and len(line) > 0 and line != '{{TOC}}':
self.description = line
elif line.startswith('* '):
m = re.match('\* \*(\w+)\*: (.*)', line)
if not m:
continue
key, value = m.group(1), m.group(2)
if member is None:
if key == 'Date':
self.date = self.parse_date(value)
elif key == 'Start' or key == 'Begin':
self.begin = self.parse_time(value)
elif key == 'End':
self.end = self.parse_time(value)
self.check_time_coherence()
elif key == 'Location':
self.location = value
else:
if key == 'Repository':
m = re.match('"(.*.git)":.*', value)
if m:
member.repository = m.group(1)
else:
member.repository = value
elif key == 'Hardware':
member.hardware = value
elif key == 'Availabilities':
member.availabilities = value
elif line.startswith('[['):
m = re.match('\[\[(\w+)\]\]\|\[\[(\w+)\]\]\|(.*)\|', line)
if not m:
print >>sys.stderr, 'Unable to parse task: "%s"' % line
continue
task = Task(m.group(1), m.group(2))
member.tasks.append(task)
if m.group(3) == '!/img/weboob/_progress.png!':
task.status = task.STATUS_PROGRESS
continue
mm = re.match('!/img/weboob/_done.png! (\d+):(\d+) (\w+)', m.group(3))
if mm and self.date:
task.status = task.STATUS_DONE
task.date = datetime(self.date.year,
self.date.month,
self.date.day,
int(mm.group(1)),
int(mm.group(2)))
task.branch = mm.group(3)
def parse_date(self, value):
try:
return datetime.strptime(value, '%Y-%m-%d')
except ValueError:
return None
def parse_time(self, value):
m = re.match('(\d+):(\d+)', value)
if not m:
return
try:
return self.date.replace(hour=int(m.group(1)),
minute=int(m.group(2)))
except ValueError:
return None
def save(self, message):
if self.winner:
finished = u'\nh3=. Event finished. Winner is "%s":/users/%d!\n' % (self.winner.name,
self.winner.id)
else:
finished = u''
s = u"""h1. %s
{{TOC}}
%s
h2. Event
%s
* *Date*: %s
* *Begin*: %s
* *End*: %s
* *Duration*: %s
* *Location*: %s
h2. Attendees
""" % (self.title,
finished,
self.description,
self.date.strftime('%Y-%m-%d') if self.date else '_Unknown_',
self.begin.strftime('%H:%M') if self.begin else '_Unknown_',
self.end.strftime('%H:%M') if self.end else '_Unknown_',
self.format_duration() or '_Unknown_',
self.location or '_Unknown_')
for member in self.members.itervalues():
if self.date:
availabilities = ''
else:
availabilities = '* *Availabilities*: %s' % member.availabilities
if member.repository is None:
repository = '_Unknown_'
elif member.repository.endswith('.git'):
repository = '"%s":git://git.symlink.me/pub/%s ("http":http://git.symlink.me/?p=%s;a=summary)'
repository = repository.replace('%s', member.repository)
else:
repository = member.repository
s += u"""h3. "%s":/users/%d
* *Repository*: %s
* *Hardware*: %s
%s
|_.Backend|_.Capabilities|_.Status|""" % (member.name,
member.id,
repository,
member.hardware,
availabilities)
for task in member.tasks:
if task.status == task.STATUS_DONE:
status = '!/img/weboob/_done.png! %02d:%02d %s' % (task.date.hour,
task.date.minute,
task.branch)
elif task.status == task.STATUS_PROGRESS:
status = '!/img/weboob/_progress.png!'
else:
status = ' '
s += u"""
|=.!/img/weboob/%s.png!:/projects/weboob/wiki/%s
[[%s]]|[[%s]]|%s|""" % (task.backend.lower(), task.backend, task.backend, task.capability, status)
s += '\n\n'
self.content.content = s
self.backend.push_content(self.content, message)
class Boobathon(ReplApplication):
APPNAME = 'boobathon'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2011 Romain Bignon'
DESCRIPTION = 'Console application to participate to a Boobathon.'
CAPS = ICapContent
SYNOPSIS = 'Usage: %prog [-dqv] [-b backends] [-cnfs] boobathon\n'
SYNOPSIS += ' %prog [--help] [--version]'
radios = []
def __init__(self, *args, **kwargs):
ReplApplication.__init__(self, *args, **kwargs)
def main(self, argv):
if len(argv) < 2:
print >>sys.stderr, 'Please give the name of the boobathon'
return 1
self.event = Event(argv[1], choice(self.weboob.backend_instances.values()))
if self.event.description is None:
if not self.ask("This event doesn't seem to exist. Do you want to create it?", default=True):
return 1
self.edit_event()
self.save_event('Event created')
return ReplApplication.main(self, [argv[0]])
def save_event(self, message):
if self.ask("Do you confirm your changes?", default=True):
self.event.save(message)
return True
return False
def edit_event(self):
self.event.title = self.ask('Enter a title', default=self.event.title)
self.event.description = self.ask('Enter a description', default=self.event.description)
self.event.date = self.ask('Enter a date (yyyy-mm-dd)',
default=self.event.date.strftime('%Y-%m-%d') if self.event.date else '',
regexp='^(\d{4}-\d{2}-\d{2})?$')
if self.event.date:
self.event.date = datetime.strptime(self.event.date, '%Y-%m-%d')
s = self.ask('Begin at (HH:MM)',
default=self.event.begin.strftime('%H:%M') if self.event.begin else '',
regexp='^(\d{2}:\d{2})?$')
if s:
h, m = s.split(':')
self.event.begin = self.event.date.replace(hour=int(h), minute=int(m))
s = self.ask('End at (HH:MM)',
default=self.event.end.strftime('%H:%M') if self.event.end else '',
regexp='^(\d{2}:\d{2})?$')
if s:
h, m = s.split(':')
self.event.end = self.event.date.replace(hour=int(h), minute=int(m))
self.event.check_time_coherence()
self.event.location = self.ask('Enter a location', default=self.event.location)
def edit_member(self, member):
if member.name is None:
firstname = self.ask('Enter your firstname')
lastname = self.ask('Enter your lastname')
member.name = '%s %s' % (firstname, lastname)
else:
member.name = self.ask('Enter your name', default=member.name)
if self.event.date is None:
member.availabilities = self.ask('Enter availabilities', default=member.availabilities)
member.repository = self.ask('Enter your repository (ex. romain/weboob.git)', default=member.repository)
member.hardware = self.ask('Enter your hardware', default=member.hardware)
def do_progress(self, line):
"""
progress
Display progress of members.
"""
self.event.load()
for member in self.event.members.itervalues():
if member.is_me and member is self.event.winner:
status = '\o/ ->'
elif member.is_me:
status = ' ->'
elif member is self.event.winner:
status = ' \o/'
else:
status = ' '
s = u' %s%20s %s|' % (status, member.shortname(), self.BOLD)
for task in member.tasks:
if task.status == task.STATUS_DONE:
s += '##'
elif task.status == task.STATUS_PROGRESS:
s += u'=>'
else:
s += ' '
s += '|%s' % self.NC
print s
print ''
now = datetime.now()
if self.event.begin > now:
d = self.event.begin - now
msg = 'The event will start in %d days, %02d:%02d:%02d'
elif self.event.end < now:
d = now - self.event.end
msg = 'The event is finished since %d days, %02d:%02d:%02d'
else:
tot = (self.event.end - self.event.begin).seconds
cur = (datetime.now() - self.event.begin).seconds
pct = cur*20/tot
progress = ''
for i in xrange(20):
if i < pct:
progress += '='
elif i == pct:
progress += '>'
else:
progress += ' '
print 'Event started: %s |%s| %s' % (self.event.begin.strftime('%H:%M'),
progress,
self.event.end.strftime('%H:%M'))
d = self.event.end - now
msg = 'The event will be finished in %d days, %02d:%02d:%02d'
print msg % (d.days, d.seconds/3600, d.seconds%3600/60, d.seconds%60)
def do_tasks(self, line):
"""
tasks
Display all tasks of members.
"""
self.event.load()
stop = False
i = -2
while not stop:
if i >= 0 and not i%2:
sys.stdout.write(' #%-2d' % (i/2))
else:
sys.stdout.write(' ')
if i >= 0 and i%2:
# second line of task, see if we'll stop
stop = True
for mem in self.event.members.itervalues():
if len(mem.tasks) > (i/2+1):
# there are more tasks, don't stop now
stop = False
if i == -2:
sys.stdout.write(' %s%-20s%s' % (self.BOLD, mem.shortname().encode('utf-8'), self.NC))
elif i == -1:
sys.stdout.write(' %s%-20s%s' % (self.BOLD, '-' * len(mem.shortname()), self.NC))
elif len(mem.tasks) <= (i/2):
sys.stdout.write(' ' * (20+1))
else:
task = mem.tasks[i/2]
if task.status == task.STATUS_DONE:
status = u'#'
elif task.status == task.STATUS_PROGRESS:
if not i%2:
status = u'|' #1st line
else:
status = u'v' #2nd line
else:
status = u' '
if not i%2: #1st line
line = u'%s %s' % (status, task.backend)
else: #2nd line
line = u'%s `-%s' % (status, task.capability[3:])
sys.stdout.write((u' %-20s' % line).encode('utf-8'))
sys.stdout.write('\n')
i += 1
def complete_close(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
self.event.load()
return [member.name for member in self.event.members.itervalues()]
def do_close(self, name):
"""
close WINNER
Close the event and set the winner.
"""
self.event.load()
for member in self.event.members.itervalues():
if member.name == name:
self.winner = member
if self.save_event('Close event'):
print 'Event is now closed. Winner is %s!' % self.winner.name
return
print >>sys.stderr, '"%s" not found' % name
return 3
def complete_edit(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return ['event', 'me']
def do_edit(self, line):
"""
edit [event | me]
Edit information about you or about event.
"""
if not line:
print >>sys.stderr, 'Syntax: edit [event | me]'
return 2
self.event.load()
if line == 'event':
self.edit_event()
self.save_event('Event edited')
elif line == 'me':
mem = self.event.get_me()
if not mem:
print >>sys.stderr, 'You haven\'t joined the event.'
return 1
self.edit_member(mem)
self.save_event('Member edited')
else:
print >>sys.stderr, 'Unable to edit "%s"' % line
return 1
def do_info(self, line):
"""
info
Display information about this event.
"""
self.event.load()
print self.event.title
print '-' * len(self.event.title)
print self.event.description
print ''
print 'Date:', self.event.date.strftime('%Y-%m-%d') if self.event.date else 'Unknown'
print 'Begin:', self.event.begin.strftime('%H:%M') if self.event.begin else 'Unknown'
print 'End:', self.event.end.strftime('%H:%M') if self.event.end else 'Unknown'
print 'Duration:', self.event.format_duration() or 'Unknown'
print 'Location:', self.event.location or 'Unknown'
print ''
print 'There are %d members, use the "members" command to list them' % len(self.event.members)
if self.event.get_me() is None:
print 'To join this event, use the command "join".'
def do_members(self, line):
"""
members
Display members informations.
"""
self.event.load()
for member in self.event.members.itervalues():
print member.name
print '-' * len(member.name)
print 'Repository:', member.repository
if self.event.date is None:
print 'Availabilities:', member.availabilities
print 'Hardware:', member.hardware
accompl = 0
for task in member.tasks:
if task.status == task.STATUS_DONE:
accompl += 1
print '%d tasks (%d accomplished)' % (len(member.tasks), accompl)
if member is self.event.winner:
print '=== %s is the winner!' % member.name
print ''
print 'Use the "tasks" command to display all tasks'
def do_join(self, line):
"""
join
Join this event.
"""
self.event.load()
if self.event.backend.browser.get_userid() in self.event.members:
print >>sys.stderr, 'You have already joined this event.'
return 1
if self.event.is_closed():
print >>sys.stderr, "Boobathon is closed."
return 1
m = Member(self.event.backend.browser.get_userid(), None)
self.edit_member(m)
self.event.members[m.id] = m
self.save_event('Joined the event')
def do_leave(self, line):
"""
leave
Leave this event.
"""
self.event.load()
if self.event.currently_in_event():
print >>sys.stderr, 'Unable to leave during the event, loser!'
return 1
if self.event.is_closed():
print >>sys.stderr, "Boobathon is closed."
return 1
try:
self.event.members.pop(self.event.backend.browser.get_userid())
except KeyError:
print >>sys.stderr, "You have not joined this event."
return 1
else:
self.save_event('Left the event')
def do_remtask(self, line):
"""
remtask TASK_ID
Remove a task.
"""
self.event.load()
mem = self.event.get_me()
if not mem:
print >>sys.stderr, "You have not joined this event."
return 1
if self.event.is_closed():
print >>sys.stderr, "Boobathon is closed."
return 1
try:
task_id = int(line)
except ValueError:
print >>sys.stderr, 'The task ID should be a number'
return 2
try:
task = mem.tasks.pop(task_id)
except IndexError:
print >>sys.stderr, 'Unable to find task #%d' % task_id
return 1
else:
print 'Removing task #%d (%s,%s).' % (task_id, task.backend, task.capability)
self.save_event('Remove task')
def do_addtask(self, line):
"""
addtask BACKEND CAPABILITY
Add a new task.
"""
self.event.load()
mem = self.event.get_me()
if not mem:
print >>sys.stderr, "You have not joined this event."
return 1
if self.event.is_closed():
print >>sys.stderr, "Boobathon is closed."
return 1
backend, capability = self.parse_command_args(line, 2, 2)
if not backend[0].isupper():
print >>sys.stderr, 'The backend name "%s" needs to start with a capital.' % backend
return 2
if not capability.startswith('Cap') or not capability[3].isupper():
print >>sys.stderr, '"%s" is not a proper capability name (must start with Cap).' % capability
return 2
for task in mem.tasks:
if (task.backend,task.capability) == (backend,capability):
print >>sys.stderr, "A task already exists for that."
return 1
task = Task(backend, capability)
mem.tasks.append(task)
self.save_event('New task')
def do_start(self, line):
"""
start [TASK_ID]
Start a task. If you don't give a task ID, the first available
task will be taken.
"""
self.event.load()
mem = self.event.get_me()
if not mem:
print >>sys.stderr, "You have not joined this event."
return 1
if len(mem.tasks) == 0:
print >>sys.stderr, "You don't have any task to do."
return 1
if not self.event.currently_in_event():
print >>sys.stderr, "You can't start a task, we are not in event."
return 1
if line.isdigit():
task_id = int(line)
else:
task_id = -1
last_done = -1
for i, task in enumerate(mem.tasks):
if task.status == task.STATUS_DONE:
last_done = i
elif task.status == task.STATUS_PROGRESS:
task.status = task.STATUS_NONE
print 'Task #%s (%s,%s) canceled.' % (i, task.backend, task.capability)
if (i == task_id or task_id < 0) and task.status == task.STATUS_NONE:
break
else:
print >>sys.stderr, 'Task not found.'
return 3
if task.status == task.STATUS_DONE:
print >>sys.stderr, 'Task is already done.'
return 1
task.status = task.STATUS_PROGRESS
mem.tasks.remove(task)
mem.tasks.insert(last_done + 1, task)
self.save_event('Started a task')
def do_done(self, line):
"""
done
Set the current task as done.
"""
self.event.load()
mem = self.event.get_me()
if not mem:
print >>sys.stderr, "You have not joined this event."
return 1
if self.event.is_closed():
print >>sys.stderr, "Boobathon is closed."
return 1
for i, task in enumerate(mem.tasks):
if task.status == task.STATUS_PROGRESS:
print 'Task (%s,%s) done! (%d%%)' % (task.backend, task.capability, (i+1)*100/len(mem.tasks))
if self.event.currently_in_event():
task.status = task.STATUS_DONE
task.date = datetime.now()
task.branch = self.ask('Enter name of branch')
self.save_event('Task accomplished')
else:
task.status = task.STATUS_NONE
print >>sys.stderr, 'Oops, you are out of event. Canceling the task...'
self.save_event('Cancel task')
return 1
return
print >>sys.stderr, "There isn't any task in progress."
return 1
def do_cancel(self, line):
"""
cancel
Cancel the current task.
"""
self.event.load()
mem = self.event.get_me()
if not mem:
print >>sys.stderr, "You have not joined this event."
return 1
if self.event.is_closed():
print >>sys.stderr, "Boobathon is closed."
return 1
for task in mem.tasks:
if task.status == task.STATUS_PROGRESS:
print 'Task (%s,%s) canceled.' % (task.backend, task.capability)
task.status = task.STATUS_NONE
self.save_event('Cancel task')
return
print >>sys.stderr, "There isn't any task in progress."
return 1
def load_default_backends(self):
"""
Overload a BaseApplication method.
"""
for instance_name, backend_name, params in self.weboob.backends_config.iter_backends():
if backend_name != 'redmine':
continue
v = urlsplit(params['url'])
if v.netloc == 'symlink.me':
self.load_backends(names=[instance_name])
return
if not self.check_loaded_backends({'url': 'https://symlink.me'}):
print "Ok, so leave now, fag."
sys.exit(0)
def is_backend_loadable(self, backend):
"""
Overload a ConsoleApplication method.
"""
return backend.name == 'redmine'
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/boobmsg/ 0000775 0000000 0000000 00000000000 11666415431 0026243 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/boobmsg/__init__.py 0000664 0000000 0000000 00000001426 11666415431 0030357 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .boobmsg import Boobmsg
__all__ = ['Boobmsg']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/boobmsg/boobmsg.py 0000664 0000000 0000000 00000031151 11666415431 0030246 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import sys
from weboob.core import CallErrors
from weboob.capabilities.messages import ICapMessages, Message, Thread
from weboob.capabilities.account import ICapAccount
from weboob.tools.application.repl import ReplApplication
from weboob.tools.application.formatters.iformatter import IFormatter
from weboob.tools.misc import html2text
__all__ = ['Boobmsg']
class XHtmlFormatter(IFormatter):
def flush(self):
pass
def format_dict(self, item):
result = "
\n"
result += "
%s
" % (item['title'])
result += "
"
result += "
Date
%s
" % (item['date'])
result += "
Sender
%s
" % (item['sender'])
result += "
Signature
%s
" % (item['signature'])
result += "
"
result += "
%s
" % (item['content'])
result += "
\n"
return result
class MessageFormatter(IFormatter):
def flush(self):
pass
def format_dict(self, item):
result = u'%sTitle:%s %s\n' % (self.BOLD,
self.NC, item['title'])
result += u'%sDate:%s %s\n' % (self.BOLD,
self.NC, item['date'])
result += u'%sFrom:%s %s\n' % (self.BOLD,
self.NC, item['sender'])
if item['receivers']:
result += u'%sTo:%s %s\n' % (self.BOLD,
self.NC,
', '.join(item['receivers']))
if item['flags'] & Message.IS_HTML:
content = html2text(item['content'])
else:
content = item['content']
result += '\n%s' % content
if item['signature']:
if item['flags'] & Message.IS_HTML:
signature = html2text(item['signature'])
else:
signature = item['signature']
result += '\n-- \n%s' % signature
return result
class MessagesListFormatter(IFormatter):
MANDATORY_FIELDS = ()
count = 0
_list_messages = False
def flush(self):
self.count = 0
def format_dict(self, item):
if not self._list_messages:
return self.format_dict_thread(item)
else:
return self.format_dict_messages(item)
def format_dict_thread(self, item):
self.count += 1
if item['nb_unread'] and item['nb_unread'] > 0:
unread = '[N]'
else:
unread = ' '
if self.interactive:
backend = item['id'].split('@', 1)[1]
result = u'%s* (%d) %s %s (%s)%s' % (self.BOLD,
self.count, unread,
item['title'], backend,
self.NC)
else:
result = u'%s* (%s) %s %s%s' % (self.BOLD, item['id'],
unread, item['title'],
self.NC)
if item['date']:
result += u'\n %s' % item['date']
return result
def format_dict_messages(self, item):
backend = item['id'].split('@', 1)[1]
if item['flags'] == Thread.IS_THREADS:
depth = 0
else:
depth = -1
result = self.format_message(backend, item['root'], depth)
return result
def format_message(self, backend, message, depth=0):
if not message:
return u''
self.count += 1
flags = '['
if message.flags & message.IS_UNREAD:
flags += 'N'
else:
flags += '-'
if message.flags & message.IS_NOT_ACCUSED:
flags += 'U'
elif message.flags & message.IS_ACCUSED:
flags += 'R'
else:
flags += '-'
flags += ']'
if self.interactive:
result = u'%s%s* (%d)%s %s <%s> %s (%s)\n' % (depth * ' ',
self.BOLD,
self.count,
self.NC,
flags,
message.sender,
message.title,
backend)
else:
result = u'%s%s* (%s.%s@%s)%s %s <%s> %s\n' % (depth * ' ',
self.BOLD,
message.thread.id,
message.id,
backend,
flags,
self.NC,
message.sender,
message.title)
if message.children:
if depth >= 0:
depth += 1
for m in message.children:
result += self.format_message(backend, m, depth)
return result
class Boobmsg(ReplApplication):
APPNAME = 'boobmsg'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Christophe Benz'
DESCRIPTION = "Console application allowing to send messages on various websites and " \
"to display message threads and contents."
CAPS = ICapMessages
EXTRA_FORMATTERS = {'msglist': MessagesListFormatter,
'msg': MessageFormatter,
'xhtml': XHtmlFormatter,
}
COMMANDS_FORMATTERS = {'list': 'msglist',
'show': 'msg',
'export_thread': 'msg',
'export_all': 'msg',
'ls': 'msglist',
}
def add_application_options(self, group):
group.add_option('-e', '--skip-empty', action='store_true',
help='Don\'t send messages with an empty body.')
group.add_option('-t', '--title', action='store',
help='For the "post" command, set a title to message',
type='string', dest='title')
def load_default_backends(self):
self.load_backends(ICapMessages, storage=self.create_storage())
def do_status(self, line):
"""
status
Display status information about a backend.
"""
if len(line) > 0:
backend_name = line
else:
backend_name = None
results = {}
for backend, field in self.do('get_account_status',
backends=backend_name,
caps=ICapAccount):
if backend.name in results:
results[backend.name].append(field)
else:
results[backend.name] = [field]
for name, fields in results.iteritems():
print ':: %s ::' % name
for f in fields:
if f.flags & f.FIELD_HTML:
value = html2text(f.value)
else:
value = f.value
print '%s: %s' % (f.label, value)
print ''
def do_post(self, line):
"""
post RECEIVER@BACKEND[,RECEIVER@BACKEND[...]] [TEXT]
Post a message to the specified receivers.
Multiple receivers are separated by a comma.
If no text is supplied on command line, the content of message is read on stdin.
"""
receivers, text = self.parse_command_args(line, 2, 1)
if text is None:
text = self.acquire_input()
if self.options.skip_empty and not text.strip():
return
for receiver in receivers.strip().split(','):
receiver, backend_name = self.parse_id(receiver.strip(),
unique_backend=True)
if not backend_name and len(self.enabled_backends) > 1:
self.logger.warning(u'No backend specified for receiver "%s": message will be sent with all the '
'enabled backends (%s)' % (receiver,
','.join(backend.name for backend in self.enabled_backends)))
if '.' in receiver:
# It's a reply
thread_id, parent_id = receiver.rsplit('.', 1)
else:
# It's an original message
thread_id = receiver
parent_id = None
thread = Thread(thread_id)
message = Message(thread,
0,
title=self.options.title,
parent=Message(thread, parent_id) if parent_id else None,
content=text)
try:
self.do('post_message', message, backends=backend_name).wait()
except CallErrors, errors:
self.bcall_errors_handler(errors)
else:
if self.interactive:
print 'Message sent sucessfully to %s' % receiver
threads = []
messages = []
def do_list(self, arg):
"""
list
Display all threads.
"""
if len(arg) > 0:
try:
thread = self.threads[int(arg) - 1]
except (IndexError, ValueError):
id, backend_name = self.parse_id(arg)
else:
id = thread.id
backend_name = thread.backend
self.messages = []
cmd = self.do('get_thread', id, backends=backend_name)
self.formatter._list_messages = True
else:
self.threads = []
cmd = self.do('iter_threads')
self.formatter._list_messages = False
for backend, thread in cmd:
if not thread:
continue
if len(arg) > 0:
for m in thread.iter_all_messages():
if not m.backend:
m.backend = thread.backend
self.messages.append(m)
else:
self.threads.append(thread)
self.format(thread)
self.flush()
def do_export_all(self, arg):
"""
export_all
Export All threads
"""
def func(backend):
for thread in backend.iter_threads():
if not thread:
continue
t = backend.fillobj(thread, None)
for msg in t.iter_all_messages():
yield msg
for backend, msg in self.do(func):
self.format(msg)
def do_export_thread(self, arg):
"""
export_thread
Export a thread
"""
_id, backend_name = self.parse_id(arg)
cmd = self.do('get_thread', _id, backends=backend_name)
for backend, thread in cmd:
if thread is not None :
for msg in thread.iter_all_messages():
self.format(msg)
def do_show(self, arg):
"""
show MESSAGE
Read a message
"""
if len(arg) == 0:
print >>sys.stderr, 'Please give a message ID.'
return 2
try:
message = self.messages[int(arg) - 1]
except (IndexError, ValueError):
id, backend_name = self.parse_id(arg)
else:
self.format(message)
self.weboob.do('set_message_read', message, backends=message.backend)
return
if not self.interactive:
print >>sys.stderr, 'Oops, you need to be in interactive mode to read messages'
return 1
else:
print >>sys.stderr, 'Message not found'
return 3
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/boobtracker/ 0000775 0000000 0000000 00000000000 11666415431 0027110 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/boobtracker/__init__.py 0000664 0000000 0000000 00000001433 11666415431 0031222 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .boobtracker import BoobTracker
__all__ = ['BoobTracker']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/boobtracker/boobtracker.py 0000664 0000000 0000000 00000031232 11666415431 0031760 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import timedelta
import sys
from weboob.capabilities.bugtracker import ICapBugTracker, Query, Update
from weboob.tools.application.repl import ReplApplication
from weboob.tools.application.formatters.iformatter import IFormatter
from weboob.tools.misc import html2text
__all__ = ['BoobTracker']
class IssueFormatter(IFormatter):
MANDATORY_FIELDS = ('id', 'project', 'title', 'body', 'author')
def flush(self):
pass
def format_dict(self, item):
result = u'%s%s - #%s - %s%s\n' % (self.BOLD, item['project'].name, item['id'], item['title'], self.NC)
result += '\n%s\n\n' % item['body']
result += 'Author: %s (%s)\n' % (item['author'].name, item['creation'])
if item['status']:
result += 'Status: %s\n' % item['status'].name
if item['version']:
result += 'Version: %s\n' % item['version'].name
if item['category']:
result += 'Category: %s\n' % item['category']
if item['assignee']:
result += 'Assignee: %s\n' % (item['assignee'].name)
if item['attachments']:
result += '\nAttachments:\n'
for a in item['attachments']:
result += '* %s%s%s <%s>\n' % (self.BOLD, a.filename, self.NC, a.url)
if item['history']:
result += '\nHistory:\n'
for u in item['history']:
result += '* %s%s - %s%s\n' % (self.BOLD, u.date, u.author.name, self.NC)
for change in u.changes:
result += ' - %s%s%s: %s -> %s\n' % (self.BOLD, change.field, self.NC, change.last, change.new)
if u.message:
result += html2text(u.message)
return result
class IssuesListFormatter(IFormatter):
MANDATORY_FIELDS = ('id', 'project', 'status', 'title', 'category')
count = 0
def flush(self):
self.count = 0
pass
def format_dict(self, item):
self.count += 1
result = u'%s* (%s) %s - [%s] %s%s\n' % (self.BOLD, item['id'], item['project'].name, item['status'].name, item['title'], self.NC)
result += ' %s' % (item['category'])
return result
class BoobTracker(ReplApplication):
APPNAME = 'boobtracker'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2011 Romain Bignon'
DESCRIPTION = "Console application allowing to send messages on various websites and " \
"to display message threads and contents."
CAPS = ICapBugTracker
EXTRA_FORMATTERS = {'issue_info': IssueFormatter,
'issues_list': IssuesListFormatter,
}
COMMANDS_FORMATTERS = {'get': 'issue_info',
'post': 'issue_info',
'edit': 'issue_info',
'search': 'issues_list',
'ls': 'issues_list',
}
def add_application_options(self, group):
group.add_option('--author')
group.add_option('--title')
group.add_option('--assignee')
group.add_option('--target-version', dest='version')
group.add_option('--category')
group.add_option('--status')
def do_search(self, line):
"""
search PROJECT
List issues for a project.
You can use these filters from command line:
--author AUTHOR
--title TITLE_PATTERN
--assignee ASSIGNEE
--target-version VERSION
--category CATEGORY
--status STATUS
"""
query = Query()
path = self.working_path.get()
backends = []
if line.strip():
query.project, backends = self.parse_id(line, unique_backend=True)
elif len(path) > 0:
query.project = path[0]
else:
print >>sys.stderr, 'Please enter a project name'
return 1
query.author = self.options.author
query.title = self.options.title
query.assignee = self.options.assignee
query.version = self.options.version
query.category = self.options.category
query.status = self.options.status
self.change_path('/%s/search' % query.project)
for backend, issue in self.do('iter_issues', query, backends=backends):
self.add_object(issue)
self.format(issue)
self.flush()
def complete_get(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_object()
def do_get(self, line):
"""
get ISSUE
Get an issue and display it.
"""
if not line:
print >>sys.stderr, 'This command takes an argument: %s' % self.get_command_help('get', short=True)
return 2
issue = self.get_object(line, 'get_issue')
if not issue:
print >>sys.stderr, 'Issue not found: %s' % line
return 3
self.format(issue)
self.flush()
def complete_comment(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_object()
def do_comment(self, line):
"""
comment ISSUE [TEXT]
Comment an issue. If no text is given, enter it in standard input.
"""
id, text = self.parse_command_args(line, 2, 1)
if text is None:
text = self.acquire_input()
id, backend_name = self.parse_id(id, unique_backend=True)
update = Update(0)
update.message = text
self.do('update_issue', id, update, backends=backend_name).wait()
def do_logtime(self, line):
"""
logtime ISSUE HOURS [TEXT]
Log spent time on an issue.
"""
id, hours, text = self.parse_command_args(line, 3, 2)
if text is None:
text = self.acquire_input()
try:
hours = float(hours)
except ValueError:
print >>sys.stderr, 'Error: HOURS parameter may be a float'
return 1
id, backend_name = self.parse_id(id, unique_backend=True)
update = Update(0)
update.message = text
update.hours = timedelta(hours=hours)
self.do('update_issue', id, update, backends=backend_name).wait()
def complete_remove(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_object()
def do_remove(self, line):
"""
remove ISSUE
Remove an issue.
"""
id, backend_name = self.parse_id(line, unique_backend=True)
self.do('remove_issue', id, backends=backend_name).wait()
ISSUE_FIELDS = (('title', (None, False)),
('assignee', ('members', True)),
('version', ('versions', True)),
('category', ('categories', False)),
('status', ('statuses', True)),
)
def get_list_item(self, objects_list, name):
if name is None:
return None
for obj in objects_list:
if obj.name.lower() == name.lower():
return obj
print 'Error: "%s" is not found' % name
return None
def prompt_issue(self, issue, requested_key=None, requested_value=None):
for key, (list_name, is_list_object) in self.ISSUE_FIELDS:
if requested_key and requested_key != key:
continue
if requested_value:
value = requested_value
elif not self.interactive:
value = getattr(self.options, key)
else:
value = None
if sys.stdin.isatty():
default = getattr(issue, key)
if not default:
default = None
elif 'name' in dir(default):
default = default.name
if list_name is None:
if value is not None:
setattr(issue, key, value)
print '%s: %s' % (key.capitalize(), value)
continue
setattr(issue, key, self.ask(key.capitalize(), default=default))
else:
objects_list = getattr(issue.project, list_name)
if len(objects_list) == 0:
continue
print '----------'
if value is not None:
if is_list_object:
value = self.get_list_item(objects_list, value)
if value is not None:
setattr(issue, key, value)
print '%s: %s' % (key.capitalize(), value.name)
continue
while value is None:
print 'Availables:', ', '.join([(o if isinstance(o, basestring) else o.name) for o in objects_list])
if is_list_object and getattr(issue, key):
default = getattr(issue, key).name
else:
default = getattr(issue, key) or ''
text = self.ask(key.capitalize(), default=default)
if not text:
break
if is_list_object:
value = self.get_list_item(objects_list, text)
else:
value = text
if value is not None:
setattr(issue, key, value)
def do_post(self, line):
"""
post PROJECT
Post a new issue.
If you are not in interactive mode, you can use these parameters:
--title TITLE
--assignee ASSIGNEE
--target-version VERSION
--category CATEGORY
--status STATUS
"""
if not line.strip():
print 'Please give the project name'
return 1
project, backend_name = self.parse_id(line, unique_backend=True)
backend = self.weboob.get_backend(backend_name)
issue = backend.create_issue(project)
self.prompt_issue(issue)
if sys.stdin.isatty():
print '----------'
print 'Please enter the content of this new issue.'
issue.body = self.acquire_input()
for backend, issue in self.weboob.do('post_issue', issue, backends=backend):
if issue:
print 'Issue %s%s@%s%s created' % (self.BOLD, issue.id, issue.backend, self.NC)
def complete_remove(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_object()
if len(args) == 3:
return dict(self.ISSUE_FIELDS).keys()
def do_edit(self, line):
"""
edit ISSUE [KEY [VALUE]]
Edit an issue.
If you are not in interactive mode, you can use these parameters:
--title TITLE
--assignee ASSIGNEE
--target-version VERSION
--category CATEGORY
--status STATUS
"""
_id, key, value = self.parse_command_args(line, 3, 1)
issue = self.get_object(_id, 'get_issue')
if not issue:
print >>sys.stderr, 'Issue not found: %s' % _id
return 3
self.prompt_issue(issue, key, value)
for backend, i in self.weboob.do('post_issue', issue, backends=issue.backend):
if i:
print 'Issue %s%s@%s%s updated' % (self.BOLD, issue.id, issue.backend, self.NC)
self.format(i)
self.flush()
def complete_attach(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_object()
elif len(args) >= 3:
return self.path_completer(args[2])
def do_attach(self, line):
"""
attach ISSUE FILENAME
Attach a file to an issue (Not implemented yet).
"""
print >>sys.stderr, 'Not implemented yet.'
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/chatoob/ 0000775 0000000 0000000 00000000000 11666415431 0026232 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/chatoob/__init__.py 0000664 0000000 0000000 00000001425 11666415431 0030345 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .chatoob import Chatoob
__all__ = ['Chatoob']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/chatoob/chatoob.py 0000664 0000000 0000000 00000004165 11666415431 0030231 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import logging
from weboob.tools.application.repl import ReplApplication
from weboob.capabilities.chat import ICapChat
from weboob.capabilities.contact import ICapContact, Contact
__all__ = ['Chatoob']
class Chatoob(ReplApplication):
APPNAME = 'chatoob'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Christophe Benz'
DESCRIPTION = 'Console application allowing to chat with contacts on various websites.'
CAPS = ICapChat
def on_new_chat_message(self, message):
print 'on_new_chat_message: %s' % message
def do_list(self, line):
"""
list
List all contacts.
"""
for backend, contact in self.do('iter_contacts', status=Contact.STATUS_ONLINE, caps=ICapContact):
self.format(contact)
self.flush()
def do_messages(self, line):
"""
messages
Get messages.
"""
for backend, message in self.do('iter_chat_messages'):
self.format(message)
self.flush()
def do_send(self, line):
"""
send CONTACT MESSAGE
Send a message to the specified contact.
"""
_id, message = self.parse_command_args(line, 2, 2)
for backend, result in self.do('send_chat_message', _id, message):
if not result:
logging.error(u'Failed to send message to contact id="%s" on backend "%s"' % (_id, backend.name))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/galleroob/ 0000775 0000000 0000000 00000000000 11666415431 0026561 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/galleroob/__init__.py 0000664 0000000 0000000 00000001434 11666415431 0030674 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .galleroob import Galleroob
__all__ = ['Galleroob']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/galleroob/galleroob.py 0000664 0000000 0000000 00000011253 11666415431 0031103 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import sys
import os
from re import search, sub
from weboob.tools.application.repl import ReplApplication
from weboob.capabilities.base import NotLoaded
from weboob.capabilities.gallery import ICapGallery
from weboob.tools.application.formatters.iformatter import IFormatter
__all__ = ['Galleroob']
class GalleryListFormatter(IFormatter):
MANDATORY_FIELDS = ('id', 'title')
count = 0
def flush(self):
self.count = 0
def format_dict(self, item):
result = u'%s* (%s) %s%s' % (
ReplApplication.BOLD,
item['id'],
item['title'],
ReplApplication.NC)
if item['cardinality'] is not NotLoaded:
result += u' (%d pages)' % item['cardinality']
if item['description'] is not NotLoaded:
result += u'\n %-70s' % item['description']
return result
class Galleroob(ReplApplication):
APPNAME = 'galleroob'
VERSION = '0.9.1'
COPYRIGHT = u'Copyright(C) 2011 Noé Rubinstein'
DESCRIPTION = 'galleroob browses and downloads web image galleries'
CAPS = ICapGallery
EXTRA_FORMATTERS = {'gallery_list': GalleryListFormatter}
COMMANDS_FORMATTERS = {'search': 'gallery_list'}
def __init__(self, *args, **kwargs):
ReplApplication.__init__(self, *args, **kwargs)
def do_search(self, pattern=None):
"""
search PATTERN
List galleries matching a PATTERN.
If PATTERN is not given, the command will list all the galleries
"""
self.set_formatter_header(u'Search pattern: %s' %
pattern if pattern else u'Latest galleries')
for backend, gallery in self.do('iter_search_results',
pattern=pattern, max_results=self.options.count):
self.add_object(gallery)
self.format(gallery)
def do_download(self, line):
"""
download ID [FIRST [FOLDER]]
Download a gallery.
Begins at page FIRST (default: 0) and saves to FOLDER (default: title)
"""
_id, first, dest = self.parse_command_args(line, 3, 1)
if first is None:
first = 0
else:
first = int(first)
gallery = None
_id, backend = self.parse_id(_id)
for _backend, result in self.do('get_gallery', _id, backends=backend):
if result:
backend = _backend
gallery = result
if not gallery:
print >>sys.stderr, 'Gallery not found: %s' % _id
return 3
backend.fillobj(gallery, ('title',))
if dest is None:
dest = sub('/', ' ', gallery.title)
print "Downloading to %s" % dest
try:
os.mkdir(dest)
except OSError:
pass # ignore error on existing directory
os.chdir(dest) # fail here if dest couldn't be created
i = 0
for img in backend.iter_gallery_images(gallery):
i += 1
if i < first:
continue
backend.fillobj(img, ('url','data'))
if img.data is None:
backend.fillobj(img, ('url','data'))
if img.data is None:
print >>sys.stderr, "Couldn't get page %d, exiting" % i
break
ext = search(r"\.([^\.]{1,5})$", img.url)
if ext:
ext = ext.group(1)
else:
ext = "jpg"
name = '%03d.%s' % (i, ext)
print 'Writing file %s' % name
with open(name, 'w') as f:
f.write(img.data)
os.chdir(os.path.pardir)
def do_info(self, line):
"""
info ID
Get information about a gallery.
"""
_id, = self.parse_command_args(line, 1, 1)
gallery = self.get_object(_id, 'get_gallery')
if not gallery:
print >>sys.stderr, 'Gallery not found: %s' % _id
return 3
self.format(gallery)
self.flush()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/geolooc/ 0000775 0000000 0000000 00000000000 11666415431 0026242 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/geolooc/__init__.py 0000664 0000000 0000000 00000001530 11666415431 0030352 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# vim: ft=python et softtabstop=4 cinoptions=4 shiftwidth=4 ts=4 ai
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .geolooc import Geolooc
__all__ = ['Geolooc']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/geolooc/geolooc.py 0000664 0000000 0000000 00000002517 11666415431 0030250 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import sys
from weboob.capabilities.geolocip import ICapGeolocIp
from weboob.tools.application.repl import ReplApplication
__all__ = ['Geolooc']
class Geolooc(ReplApplication):
APPNAME = 'geolooc'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
DESCRIPTION = 'Console application allowing to geolocalize IP addresses.'
CAPS = ICapGeolocIp
def main(self, argv):
if len(argv) < 2:
print >>sys.stderr, 'Syntax: %s ipaddr' % argv[0]
return 2
for backend, location in self.do('get_location', argv[1]):
self.format(location)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/havesex/ 0000775 0000000 0000000 00000000000 11666415431 0026256 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/havesex/__init__.py 0000664 0000000 0000000 00000001424 11666415431 0030370 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .havesex import HaveSex
__all__ = ['HaveSex']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/havesex/havesex.py 0000664 0000000 0000000 00000026301 11666415431 0030275 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import sys
import weboob
from weboob.tools.application.repl import ReplApplication
from weboob.tools.application.formatters.iformatter import IFormatter
from weboob.capabilities.dating import ICapDating, OptimizationNotFound
from weboob.capabilities.contact import Contact
__all__ = ['HaveSex']
class ProfileFormatter(IFormatter):
def flush(self):
pass
def print_node(self, node, level=1):
result = u''
if node.flags & node.SECTION:
result += u'\t' * level + node.label + '\n'
for sub in node.value.itervalues():
result += self.print_node(sub, level+1)
else:
if isinstance(node.value, (tuple,list)):
value = ', '.join(unicode(v) for v in node.value)
else:
value = node.value
result += u'\t' * level + u'%-20s %s\n' % (node.label + ':', value)
return result
def format_dict(self, item):
result = u'Nickname: %s\n' % item['name']
if item['status'] & Contact.STATUS_ONLINE:
s = 'online'
elif item['status'] & Contact.STATUS_OFFLINE:
s = 'offline'
elif item['status'] & Contact.STATUS_AWAY:
s = 'away'
else:
s = 'unknown'
result += u'Status: %s (%s)\n' % (s, item['status_msg'])
result += u'Photos:\n'
for name, photo in item['photos'].iteritems():
result += u'\t%s%s\n' % (photo, ' (hidden)' if photo.hidden else '')
result += u'Profile:\n'
for head in item['profile'].itervalues():
result += self.print_node(head)
result += u'Description:\n'
for s in item['summary'].split('\n'):
result += u'\t%s\n' % s
return result
class HaveSex(ReplApplication):
APPNAME = 'havesex'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
DESCRIPTION = 'Console application allowing to interact with various dating websites ' \
'and to optimize seduction algorithmically.'
STORAGE_FILENAME = 'dating.storage'
STORAGE = {'optims': {}}
CAPS = ICapDating
EXTRA_FORMATTERS = {'profile': ProfileFormatter}
COMMANDS_FORMATTERS = {'optim': 'table',
'profile': 'profile'}
def load_default_backends(self):
self.load_backends(ICapDating, storage=self.create_storage(self.STORAGE_FILENAME))
def main(self, argv):
self.load_config()
try:
self.do('init_optimizations').wait()
except weboob.core.CallErrors, e:
self.bcall_errors_handler(e)
optimizations = self.storage.get('optims')
for optim, backends in optimizations.iteritems():
self.optims('start', backends, optim, store=False)
return ReplApplication.main(self, argv)
def do_profile(self, id):
"""
profile ID
Display a profile
"""
_id, backend_name = self.parse_id(id, unique_backend=True)
found = 0
for backend, contact in self.do('get_contact', _id, backends=backend_name):
if contact:
self.format(contact)
found = 1
if not found:
self.logger.error(u'Profile not found')
else:
self.flush()
def do_query(self, id):
"""
query ID
Send a query to someone.
"""
_id, backend_name = self.parse_id(id, unique_backend=True)
for backend, query in self.do('send_query', _id, backends=backend_name):
print '%s' % query.message
def edit_optims(self, backend_names, optims_names, stop=False):
if optims_names is None:
print >>sys.stderr, 'Error: missing parameters.'
return 2
for optim_name in optims_names.split():
backends_optims = {}
for backend, optim in self.do('get_optimization', optim_name, backends=backend_names):
if optim:
backends_optims[backend.name] = optim
for backend_name, optim in backends_optims.iteritems():
if len(optim.CONFIG) == 0:
print '%s.%s does not require configuration.' % (backend_name, optim_name)
continue
was_running = optim.is_running()
if stop and was_running:
print 'Stopping %s: %s' % (optim_name, backend_name)
optim.stop()
params = optim.get_config()
if params is None:
params = {}
print 'Configuration of %s.%s' % (backend_name, optim_name)
print '-----------------%s-%s' % ('-' * len(backend_name), '-' * len(optim_name))
for key, value in optim.CONFIG.iteritems():
params[key] = self.ask(value, default=params[key] if (key in params) else value.default)
optim.set_config(params)
if stop and was_running:
print 'Starting %s: %s' % (optim_name, backend_name)
optim.start()
def optims(self, function, backend_names, optims, store=True):
if optims is None:
print >>sys.stderr, 'Error: missing parameters.'
return 2
for optim_name in optims.split():
try:
if store:
storage_optim = set(self.storage.get('optims', optim_name, default=[]))
sys.stdout.write('%sing %s:' % (function.capitalize(), optim_name))
for backend, optim in self.do('get_optimization', optim_name, backends=backend_names):
if optim:
# It's useless to start a started optim, or to stop a stopped one.
if (function == 'start' and optim.is_running()) or \
(function == 'stop' and not optim.is_running()):
continue
# Optim is not configured and would be, ask user to do it.
if function == 'start' and len(optim.CONFIG) > 0 and optim.get_config() is None:
self.edit_optims(backend.name, optim_name)
ret = getattr(optim, function)()
sys.stdout.write(' ' + backend.name)
if not ret:
sys.stdout.write('(failed)')
sys.stdout.flush()
if store:
if function == 'start' and ret:
storage_optim.add(backend.name)
elif function == 'stop':
try:
storage_optim.remove(backend.name)
except KeyError:
pass
sys.stdout.write('.\n')
except weboob.core.CallErrors, errors:
for backend, error, backtrace in errors:
if isinstance(error, OptimizationNotFound):
self.logger.error(u'Error(%s): Optimization "%s" not found' % (backend.name, optim_name))
else:
self.bcall_error_handler(backend, error, backtrace)
if store:
if len(storage_optim) > 0:
self.storage.set('optims', optim_name, list(storage_optim))
else:
self.storage.delete('optims', optim_name)
if store:
self.storage.save()
def complete_optim(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return ['list', 'start', 'stop', 'edit']
elif len(args) == 3:
return [backend.name for backend in self.enabled_backends]
elif len(args) >= 4:
if args[2] == '*':
backend = None
else:
backend = args[2]
optims = set()
for backend, (name, optim) in self.do('iter_optimizations', backends=backend):
optims.add(name)
return sorted(optims - set(args[3:]))
def do_optim(self, line):
"""
optim [list | start | edit | stop] BACKEND [OPTIM1 [OPTIM2 ...]]
All dating backends offer optimization services. This command can be
manage them.
Use * us BACKEND value to apply command to all backends.
Commands:
* list list all available optimizations of a backend
* start start optimization services on a backend
* edit configure an optimization service for a backend
* stop stop optimization services on a backend
"""
cmd, backend_name, optims_names = self.parse_command_args(line, 3)
if backend_name == '*':
backend_name = None
elif backend_name is not None and not backend_name in [b.name for b in self.enabled_backends]:
print >>sys.stderr, 'Error: No such backend "%s"' % backend_name
return 1
if cmd == 'start':
return self.optims('start', backend_name, optims_names)
if cmd == 'stop':
return self.optims('stop', backend_name, optims_names)
if cmd == 'edit':
self.edit_optims(backend_name, optims_names, stop=True)
return
if cmd == 'list' or cmd is None:
if optims_names is not None:
optims_names = optims_names.split()
optims = {}
backends = set()
for backend, (name, optim) in self.do('iter_optimizations', backends=backend_name):
if optims_names is not None and not name in optims_names:
continue
if optim.is_running():
status = 'RUNNING'
else:
status = '-------'
if not name in optims:
optims[name] = {backend.name: status}
else:
optims[name][backend.name] = status
backends.add(backend.name)
backends = sorted(backends)
for name, backends_status in optims.iteritems():
line = [('name', name)]
for b in backends:
try:
status = backends_status[b]
except KeyError:
status = ''
line.append((b, status))
self.format(tuple(line))
self.flush()
return
print >>sys.stderr, "No such command '%s'" % cmd
return 1
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/masstransit/ 0000775 0000000 0000000 00000000000 11666415431 0027163 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/masstransit/__init__.py 0000664 0000000 0000000 00000001544 11666415431 0031300 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# vim: ft=python et softtabstop=4 cinoptions=4 shiftwidth=4 ts=4 ai
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .masstransit import Masstransit
__all__ = ['Masstransit']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/masstransit/masstransit.py 0000664 0000000 0000000 00000024163 11666415431 0032113 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Hébert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.travel import ICapTravel
from weboob.tools.application.base import BaseApplication
from logging import warning
import gtk
class FakeConic(object):
STATUS_CONNECTED = None
STATUS_DISCONNECTED = None
CONNECT_FLAG_NONE = None
def Connection(self):
raise NotImplementedError()
try:
import hildon
except ImportError:
toolkit = gtk
else:
toolkit = hildon
try :
import conic
except ImportError:
warning("conic is not found")
conic = FakeConic()
from logging import debug
__all__ = ['Masstransit']
class MasstransitHildon():
"hildon interface"
def connect_event(self, connection, event=None, c=None, d=None):
debug("DBUS-DEBUG a: %s, b:%s, c:%s,d: %s" % (connection, event, c, d))
status = event.get_status()
if status == conic.STATUS_CONNECTED:
self.connected = True
if self.touch_selector_entry_filled == False:
debug("connected, now fill")
self.fill_touch_selector_entry()
if self.refresh_in_progress:
self.refresh()
elif status == conic.STATUS_DISCONNECTED:
self.connected = False
def __init__(self, weboob):
self.touch_selector_entry_filled = False
self.refresh_in_progress = False
self.connected = False
self.weboob = weboob
try :
self.connection = conic.Connection()
self.connection.connect("connection-event", self.connect_event)
self.connection.set_property("automatic-connection-events", True)
self.connection.request_connection(conic.CONNECT_FLAG_NONE)
except NotImplementedError:
pass
horizontal_box = gtk.HBox()
self.main_window = toolkit.Window()
try :
self.refresh_button = toolkit.Button(
gtk.HILDON_SIZE_AUTO_WIDTH | gtk.HILDON_SIZE_FINGER_HEIGHT,
hildon.BUTTON_ARRANGEMENT_HORIZONTAL,
"Actualiser"
)
self.retour_button = hildon.Button(
gtk.HILDON_SIZE_AUTO_WIDTH | gtk.HILDON_SIZE_FINGER_HEIGHT,
hildon.BUTTON_ARRANGEMENT_HORIZONTAL,
"Retour"
)
self.combo_source = hildon.TouchSelectorEntry(text=True)
self.combo_dest = hildon.TouchSelectorEntry(text=True)
self.picker_button_source = hildon.PickerButton(
gtk.HILDON_SIZE_AUTO,
hildon.BUTTON_ARRANGEMENT_VERTICAL)
self.picker_button_dest = hildon.PickerButton(
gtk.HILDON_SIZE_AUTO,
hildon.BUTTON_ARRANGEMENT_VERTICAL
)
self.picker_button_source.set_sensitive(False)
self.picker_button_dest.set_sensitive(False)
self.picker_button_source.set_title("Gare de Depart")
self.picker_button_dest.set_title("Gare d'arrivee")
self.picker_button_source.set_selector(self.combo_source)
self.picker_button_dest.set_selector(self.combo_dest)
horizontal_box.pack_start(self.picker_button_source)
horizontal_box.pack_start(self.picker_button_dest)
except AttributeError:
self.refresh_button = gtk.Button("Actualiser")
self.retour_button = gtk.Button("Retour")
self.combo_source = gtk.combo_box_entry_new_text()
self.combo_dest = gtk.combo_box_entry_new_text()
horizontal_box.pack_start(self.combo_source)
horizontal_box.pack_start(self.combo_dest)
self.main_window.set_title("Horaires des Prochains Trains")
self.main_window.connect("destroy", self.on_main_window_destroy)
self.refresh_button.connect("clicked", self.on_refresh_button_clicked)
self.retour_button.set_sensitive(False)
self.retour_button.connect("clicked", self.on_retour_button_clicked)
self.treestore = gtk.TreeStore(str, str, str, str, str)
treeview = gtk.TreeView(self.treestore)
treeview.append_column(
gtk.TreeViewColumn(
'Train',
gtk.CellRendererText(),
text=0
))
treeview.append_column(
gtk.TreeViewColumn(
'Horaire',
gtk.CellRendererText(),
text=1
))
treeview.append_column(
gtk.TreeViewColumn(
'Destination',
gtk.CellRendererText(),
text=2
))
treeview.append_column(
gtk.TreeViewColumn(
'Voie',
gtk.CellRendererText(),
text=3
))
treeview.append_column(
gtk.TreeViewColumn(
'Information',
gtk.CellRendererText(),
text=4
))
vertical_box = gtk.VBox()
vertical_box.pack_start(horizontal_box)
horizontal_box.pack_start(self.retour_button)
vertical_box.pack_start(treeview)
vertical_box.pack_start(self.refresh_button)
self.main_window.add(vertical_box)
self.main_window.show_all()
self.fill_touch_selector_entry()
if toolkit != gtk:
self.picker_button_source.connect("value-changed",
self.check_station_input,
self.picker_button_source)
self.picker_button_dest.connect("value-changed",
self.check_station_input,
self.picker_button_dest)
def fill_touch_selector_entry(self):
liste = []
for backend in self.weboob.iter_backends():
for station in backend.iter_station_search(""):
liste.append(station.name.capitalize())
liste.sort()
for station in liste:
self.combo_source.append_text(station)
self.combo_dest.append_text(station)
self.touch_selector_entry_filled = True
if toolkit != gtk:
self.picker_button_source.set_sensitive(True)
def on_main_window_destroy(self, widget):
"exit application at the window close"
gtk.main_quit()
def on_main_window_show(self, param):
self.fill_touch_selector_entry()
def on_retour_button_clicked(self, widget):
"the button is clicked"
debug("on_retour_button_clicked")
self.refresh_in_progress = True
col_source = self.combo_source.get_active(0)
col_dest = self.combo_dest.get_active(0)
self.combo_source.set_active(0, col_dest)
self.combo_dest.set_active(0, col_source)
self.refresh()
def on_refresh_button_clicked(self, widget):
"the refresh button is clicked"
debug("on_refresh_button_clicked")
self.refresh_in_progress = True
try :
self.connection.request_connection(conic.CONNECT_FLAG_NONE)
except AttributeError:
if isinstance(conic, FakeConic):
self.refresh()
else:
raise
def check_station_input(self, widget, user_data):
if self.combo_source.get_current_text() is None :
self.picker_button_dest.set_sensitive(False)
self.refresh_button.set_sensitive(False)
self.retour_button.set_sensitive(False)
else:
self.picker_button_dest.set_sensitive(True)
if self.combo_dest.get_current_text() is None:
self.refresh_button.set_sensitive(False)
self.retour_button.set_sensitive(False)
else:
self.refresh_button.set_sensitive(True)
self.retour_button.set_sensitive(True)
def refresh(self):
"update departures"
banner = hildon.hildon_banner_show_information(self.main_window, "", "Chargement en cours")
banner.set_timeout(10000)
hildon.hildon_gtk_window_set_progress_indicator(self.main_window, 1)
self.treestore.clear()
try :
source_text = self.combo_source.get_current_text()
dest_text = self.combo_dest.get_current_text()
except AttributeError:
source_text = self.combo_source.child.get_text()
dest_text = self.combo_dest.child.get_text()
for backend in self.weboob.iter_backends():
for station in backend.iter_station_search(source_text):
for arrival in \
backend.iter_station_search(dest_text):
for departure in \
backend.iter_station_departures(station.id, arrival.id):
self.treestore.append(None,
[departure.type,
departure.time,
departure.arrival_station,
departure.plateform,
departure.information])
self.refresh_in_progress = False
banner.set_timeout(1)
hildon.hildon_gtk_window_set_progress_indicator(self.main_window, 0)
class Masstransit(BaseApplication):
"Application Class"
APPNAME = 'masstransit'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Julien Hébert'
def main(self, argv):
self.load_backends(ICapTravel)
MasstransitHildon(self.weboob)
gtk.main()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/monboob/ 0000775 0000000 0000000 00000000000 11666415431 0026246 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/monboob/__init__.py 0000664 0000000 0000000 00000001424 11666415431 0030360 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .monboob import Monboob
__all__ = ['Monboob']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/monboob/monboob.py 0000664 0000000 0000000 00000031423 11666415431 0030256 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon, Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from email.mime.text import MIMEText
from smtplib import SMTP
from email.Header import Header, decode_header
from email.Utils import parseaddr, formataddr, formatdate
from email import message_from_file, message_from_string
from smtpd import SMTPServer
import time
import re
import sys
import logging
import asyncore
import subprocess
import socket
from weboob.core import Weboob, CallErrors
from weboob.core.scheduler import Scheduler
from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Thread, Message
from weboob.tools.application.repl import ReplApplication
from weboob.tools.misc import html2text, get_backtrace, utc2local, to_unicode
__all__ = ['Monboob']
class FakeSMTPD(SMTPServer):
def __init__(self, app, bindaddr, port):
SMTPServer.__init__(self, (bindaddr, port), None)
self.app = app
def process_message(self, peer, mailfrom, rcpttos, data):
msg = message_from_string(data)
self.app.process_incoming_mail(msg)
class MonboobScheduler(Scheduler):
def __init__(self, app):
Scheduler.__init__(self)
self.app = app
def run(self):
if self.app.options.smtpd:
if ':' in self.app.options.smtpd:
host, port = self.app.options.smtpd.split(':', 1)
else:
host = '127.0.0.1'
port = self.app.options.smtpd
try:
FakeSMTPD(self.app, host, int(port))
except socket.error, e:
self.logger.error('Unable to start the SMTP daemon: %s' % e)
return False
# XXX Fuck, we shouldn't copy this piece of code from
# weboob.scheduler.Scheduler.run().
try:
while 1:
self.stop_event.wait(0.1)
if self.app.options.smtpd:
asyncore.loop(timeout=0.1, count=1)
except KeyboardInterrupt:
self._wait_to_stop()
raise
else:
self._wait_to_stop()
return True
class Monboob(ReplApplication):
APPNAME = 'monboob'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
DESCRIPTION = 'Daemon allowing to regularly check for new messages on various websites, ' \
'and send an email for each message, and post a reply to a message on a website.'
CONFIG = {'interval': 300,
'domain': 'weboob.example.org',
'recipient': 'weboob@example.org',
'smtp': 'localhost',
'pipe': '',
'html': 0}
CAPS = ICapMessages
DISABLE_REPL = True
def add_application_options(self, group):
group.add_option('-S', '--smtpd', help='run a fake smtpd server and set the port')
def create_weboob(self):
return Weboob(scheduler=MonboobScheduler(self))
def load_default_backends(self):
self.load_backends(ICapMessages, storage=self.create_storage())
def main(self, argv):
self.load_config()
try:
self.config.set('interval', int(self.config.get('interval')))
if self.config.get('interval') < 1:
raise ValueError()
except ValueError:
print >>sys.stderr, 'Configuration error: interval must be an integer >0.'
return 1
try:
self.config.set('html', int(self.config.get('html')))
if self.config.get('html') not in (0,1):
raise ValueError()
except ValueError:
print >>sys.stderr, 'Configuration error: html must be 0 or 1.'
return 2
return ReplApplication.main(self, argv)
def get_email_address_ident(self, msg, header):
s = msg.get(header)
if not s:
return None
m = re.match('.*<([^@]*)@(.*)>', s)
if m:
return m.group(1)
else:
try:
return s.split('@')[0]
except IndexError:
return s
def do_post(self, line):
"""
post
Pipe with a mail to post message.
"""
msg = message_from_file(sys.stdin)
return self.process_incoming_mail(msg)
def process_incoming_mail(self, msg):
to = self.get_email_address_ident(msg, 'To')
sender = msg.get('From')
reply_to = self.get_email_address_ident(msg, 'In-Reply-To')
title = msg.get('Subject')
if title:
new_title = u''
for part in decode_header(title):
if part[1]:
new_title += unicode(part[0], part[1])
else:
new_title += unicode(part[0])
title = new_title
content = u''
for part in msg.walk():
if part.get_content_type() == 'text/plain':
s = part.get_payload(decode=True)
charsets = part.get_charsets() + msg.get_charsets()
for charset in charsets:
try:
if charset is not None:
content += unicode(s, charset)
else:
content += unicode(s)
except UnicodeError, e:
self.logger.warning('Unicode error: %s' % e)
continue
except Exception, e:
self.logger.exception(e)
continue
else:
break
if len(content) == 0:
print >>sys.stderr, 'Unable to send an empty message'
return 1
# remove signature
content = content.split(u'\n-- \n')[0]
parent_id = None
if reply_to is None:
# This is a new message
if '.' in to:
bname, thread_id = to.split('.', 1)
else:
bname = to
thread_id = None
else:
# This is a reply
try:
bname, id = reply_to.split('.', 1)
thread_id, parent_id = id.rsplit('.', 1)
except ValueError:
print >>sys.stderr, 'In-Reply-To header might be in form '
return 1
# Default use the To header field to know the backend to use.
if to and bname != to:
bname = to
try:
backend = self.weboob.backend_instances[bname]
except KeyError:
print >>sys.stderr, 'Backend %s not found' % bname
return 1
if not backend.has_caps(ICapMessagesPost):
print >>sys.stderr, 'The backend %s does not implement ICapMessagesPost' % bname
return 1
thread = Thread(thread_id)
message = Message(thread,
0,
title=title,
sender=sender,
receivers=[to],
parent=Message(thread, parent_id) if parent_id else None,
content=content)
try:
backend.post_message(message)
except Exception, e:
content = u'Unable to send message to %s:\n' % thread_id
content += u'\n\t%s\n' % to_unicode(e)
if logging.root.level == logging.DEBUG:
content += u'\n%s\n' % to_unicode(get_backtrace(e))
self.send_email(backend, Message(thread,
0,
title='Unable to send message',
sender='Monboob',
parent=Message(thread, parent_id) if parent_id else None,
content=content))
def do_run(self, line):
"""
run
Run the fetching daemon.
"""
self.weboob.repeat(self.config.get('interval'), self.process)
self.weboob.loop()
def process(self):
try:
for backend, message in self.weboob.do('iter_unread_messages'):
if self.send_email(backend, message):
backend.set_message_read(message)
except CallErrors, e:
self.bcall_errors_handler(e)
def send_email(self, backend, mail):
domain = self.config.get('domain')
recipient = self.config.get('recipient')
reply_id = ''
if mail.parent:
reply_id = u'<%s.%s@%s>' % (backend.name, mail.parent.full_id, domain)
subject = mail.title
sender = u'"%s" <%s@%s>' % (mail.sender.replace('"', '""') if mail.sender else '',
backend.name, domain)
# assume that .date is an UTC datetime
date = formatdate(time.mktime(utc2local(mail.date).timetuple()), localtime=True)
msg_id = u'<%s.%s@%s>' % (backend.name, mail.full_id, domain)
if self.config.get('html') and mail.flags & mail.IS_HTML:
body = mail.content
content_type = 'html'
else:
if mail.flags & mail.IS_HTML:
body = html2text(mail.content)
else:
body = mail.content
content_type = 'plain'
if body is None:
body = ''
if mail.signature:
if self.config.get('html') and mail.flags & mail.IS_HTML:
body += u'
-- %s
' % mail.signature
else:
body += u'\n\n-- \n'
if mail.flags & mail.IS_HTML:
body += html2text(mail.signature)
else:
body += mail.signature
# Header class is smart enough to try US-ASCII, then the charset we
# provide, then fall back to UTF-8.
header_charset = 'ISO-8859-1'
# We must choose the body charset manually
for body_charset in 'US-ASCII', 'ISO-8859-1', 'UTF-8':
try:
body.encode(body_charset)
except UnicodeError:
pass
else:
break
# Split real name (which is optional) and email address parts
sender_name, sender_addr = parseaddr(sender)
recipient_name, recipient_addr = parseaddr(recipient)
# We must always pass Unicode strings to Header, otherwise it will
# use RFC 2047 encoding even on plain ASCII strings.
sender_name = str(Header(unicode(sender_name), header_charset))
recipient_name = str(Header(unicode(recipient_name), header_charset))
# Make sure email addresses do not contain non-ASCII characters
sender_addr = sender_addr.encode('ascii')
recipient_addr = recipient_addr.encode('ascii')
# Create the message ('plain' stands for Content-Type: text/plain)
msg = MIMEText(body.encode(body_charset), content_type, body_charset)
msg['From'] = formataddr((sender_name, sender_addr))
msg['To'] = formataddr((recipient_name, recipient_addr))
msg['Subject'] = Header(unicode(subject), header_charset)
msg['Message-Id'] = msg_id
msg['Date'] = date
if reply_id:
msg['In-Reply-To'] = reply_id
self.logger.info('Send mail from <%s> to <%s>' % (sender, recipient))
if len(self.config.get('pipe')) > 0:
p = subprocess.Popen(self.config.get('pipe'),
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
p.stdin.write(msg.as_string())
p.stdin.close()
if p.wait() != 0:
self.logger.error('Unable to deliver mail: %s' % p.stdout.read().strip())
return False
else:
# Send the message via SMTP to localhost:25
try:
smtp = SMTP(self.config.get('smtp'))
smtp.sendmail(sender, recipient, msg.as_string())
except Exception, e:
self.logger.error('Unable to deliver mail: %s' % e)
return False
else:
smtp.quit()
return True
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/pastoob/ 0000775 0000000 0000000 00000000000 11666415431 0026262 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/pastoob/__init__.py 0000664 0000000 0000000 00000001527 11666415431 0030400 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# vim: ft=python et softtabstop=4 cinoptions=4 shiftwidth=4 ts=4 ai
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .pastoob import Pastoob
__all__ = ['Pastoob']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/pastoob/pastoob.py 0000664 0000000 0000000 00000007133 11666415431 0030307 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
import os
import sys
import codecs
import locale
from random import choice
from weboob.capabilities.paste import ICapPaste, PasteNotFound
from weboob.tools.application.repl import ReplApplication
__all__ = ['Pastoob']
class Pastoob(ReplApplication):
APPNAME = 'pastoob'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2011 Laurent Bachelier'
DESCRIPTION = 'Console application allowing to post and get pastes from pastebins.'
CAPS = ICapPaste
def main(self, argv):
self.load_config()
return ReplApplication.main(self, argv)
def do_get(self, _id):
"""
get ID
Get a paste contents.
"""
if not _id:
print >>sys.stderr, 'This command takes an argument: %s' % self.get_command_help('get', short=True)
return 2
try:
paste = self.get_object(_id, 'get_paste', ['contents'])
except PasteNotFound:
print >>sys.stderr, 'Paste not found: %s' % _id
return 3
if not paste:
print >>sys.stderr, 'Unable to handle paste: %s' % _id
return 1
output = codecs.getwriter(sys.stdout.encoding or locale.getpreferredencoding())(sys.stdout)
output.write(paste.contents)
# add a newline unless we are writing
# in a file or in a pipe
if os.isatty(output.fileno()):
output.write('\n')
def do_post(self, filename):
"""
post [FILENAME]
Submit a new paste.
The filename can be '-' for reading standard input (pipe).
"""
if not filename or filename == '-':
contents = self.acquire_input()
else:
try:
with codecs.open(filename, encoding=locale.getpreferredencoding()) as fp:
contents = fp.read()
except IOError, e:
print >>sys.stderr, 'Unable to open file "%s": %s' % (filename, e.strerror)
return 1
# get and sort the backends able to satisfy our requirements
params = self._get_params()
backends = {}
for backend in self.weboob.iter_backends():
score = backend.can_post(contents, **params)
if score:
backends.setdefault(score, []).append(backend)
# select a random backend from the best scores
if len(backends):
backend = choice(backends[max(backends.keys())])
else:
print >>sys.stderr, 'No suitable backend found.'
return 1
p = backend.new_paste(_id=None)
p.public = params.get('public')
p.title = os.path.basename(filename)
p.contents = contents
backend.post_paste(p, max_age=params.get('max_age'))
print 'Successfuly posted paste: %s' % p.page_url
def _get_params(self):
return {'public': True, 'max_age': 3600*24*3}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qboobmsg/ 0000775 0000000 0000000 00000000000 11666415431 0026424 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qboobmsg/__init__.py 0000664 0000000 0000000 00000000067 11666415431 0030540 0 ustar 00root root 0000000 0000000 from .qboobmsg import QBoobMsg
__all__ = ['QBoobMsg']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qboobmsg/main_window.py 0000664 0000000 0000000 00000003600 11666415431 0031310 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from PyQt4.QtCore import SIGNAL
from weboob.tools.application.qt import QtMainWindow
from weboob.tools.application.qt.backendcfg import BackendCfg
from weboob.capabilities.messages import ICapMessages
from .ui.main_window_ui import Ui_MainWindow
from .messages_manager import MessagesManager
class MainWindow(QtMainWindow):
def __init__(self, config, weboob, parent=None):
QtMainWindow.__init__(self, parent)
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
self.config = config
self.weboob = weboob
self.manager = MessagesManager(weboob, self)
self.setCentralWidget(self.manager)
self.connect(self.ui.actionBackends, SIGNAL("triggered()"), self.backendsConfig)
self.connect(self.ui.actionRefresh, SIGNAL("triggered()"), self.refresh)
if self.weboob.count_backends() == 0:
self.backendsConfig()
else:
self.centralWidget().load()
def backendsConfig(self):
bckndcfg = BackendCfg(self.weboob, (ICapMessages,), self)
if bckndcfg.run():
self.centralWidget().load()
def refresh(self):
self.centralWidget().refreshThreads()
messages_manager.py 0000664 0000000 0000000 00000023312 11666415431 0032221 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qboobmsg # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import time
import logging
from PyQt4.QtGui import QWidget, QTreeWidgetItem, QListWidgetItem, QMessageBox, QBrush
from PyQt4.QtCore import SIGNAL, Qt
from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Message
from weboob.tools.application.qt import QtDo
from weboob.tools.misc import to_unicode
from .ui.messages_manager_ui import Ui_MessagesManager
class MessagesManager(QWidget):
def __init__(self, weboob, parent=None):
QWidget.__init__(self, parent)
self.ui = Ui_MessagesManager()
self.ui.setupUi(self)
self.weboob = weboob
self.ui.backendsList.setCurrentRow(0)
self.backend = None
self.thread = None
self.message = None
self.ui.replyButton.setEnabled(False)
self.ui.replyWidget.hide()
self.connect(self.ui.backendsList, SIGNAL('itemSelectionChanged()'), self._backendChanged)
self.connect(self.ui.threadsList, SIGNAL('itemSelectionChanged()'), self._threadChanged)
self.connect(self.ui.messagesTree, SIGNAL('itemClicked(QTreeWidgetItem *, int)'), self._messageSelected)
self.connect(self.ui.messagesTree, SIGNAL('itemActivated(QTreeWidgetItem *, int)'), self._messageSelected)
self.connect(self.ui.replyButton, SIGNAL('clicked()'), self._replyPressed)
self.connect(self.ui.sendButton, SIGNAL('clicked()'), self._sendPressed)
def load(self):
self.ui.backendsList.clear()
self.ui.backendsList.addItem('(All)')
for backend in self.weboob.iter_backends():
if not backend.has_caps(ICapMessages):
continue
item = QListWidgetItem(backend.name.capitalize())
item.setData(Qt.UserRole, backend)
self.ui.backendsList.addItem(item)
self.refreshThreads()
def _backendChanged(self):
selection = self.ui.backendsList.selectedItems()
if not selection:
self.backend = None
return
self.backend = selection[0].data(Qt.UserRole).toPyObject()
self.refreshThreads()
def refreshThreads(self):
self.ui.messagesTree.clear()
self.ui.threadsList.clear()
self.hideReply()
self.ui.replyButton.setEnabled(False)
self.ui.backendsList.setEnabled(False)
self.ui.threadsList.setEnabled(False)
self.process_threads = QtDo(self.weboob, self._gotThread)
self.process_threads.do('iter_threads', backends=self.backend, caps=ICapMessages)
def _gotThread(self, backend, thread):
if not backend:
self.process_threads = None
self.ui.backendsList.setEnabled(True)
self.ui.threadsList.setEnabled(True)
return
item = QListWidgetItem(thread.title)
item.setData(Qt.UserRole, (thread.backend, thread.id))
self.ui.threadsList.addItem(item)
def _threadChanged(self):
self.ui.messagesTree.clear()
selection = self.ui.threadsList.selectedItems()
if not selection:
return
t = selection[0].data(Qt.UserRole).toPyObject()
self.refreshThreadMessages(*t)
def refreshThreadMessages(self, backend, id):
self.ui.messagesTree.clear()
self.ui.messageBody.clear()
self.ui.backendsList.setEnabled(False)
self.ui.threadsList.setEnabled(False)
self.ui.replyButton.setEnabled(False)
self.hideReply()
self.process = QtDo(self.weboob, self._gotThreadMessages)
self.process.do('get_thread', id, backends=backend)
def _gotThreadMessages(self, backend, thread):
if thread is None:
self.ui.backendsList.setEnabled(True)
self.ui.threadsList.setEnabled(True)
self.process = None
return
self.thread = thread
self.showMessage(thread.root)
self._insert_message(thread.root, self.ui.messagesTree.invisibleRootItem())
self.ui.messagesTree.expandAll()
def _insert_message(self, message, top):
item = QTreeWidgetItem(None, [message.title or '', message.sender or 'Unknown',
time.strftime('%Y-%m-%d %H:%M:%S', message.date.timetuple())])
item.setData(0, Qt.UserRole, message)
if message.flags & message.IS_UNREAD:
item.setForeground(0, QBrush(Qt.darkYellow))
item.setForeground(1, QBrush(Qt.darkYellow))
item.setForeground(2, QBrush(Qt.darkYellow))
top.addChild(item)
if message.children is not None:
for child in message.children:
self._insert_message(child, item)
def _messageSelected(self, item, column):
message = item.data(0, Qt.UserRole).toPyObject()
self.showMessage(message, item)
def showMessage(self, message, item=None):
backend = self.weboob.get_backend(message.thread.backend)
if backend.has_caps(ICapMessagesPost):
self.ui.replyButton.setEnabled(True)
self.message = message
if message.title.startswith('Re:'):
self.ui.titleEdit.setText(message.title)
else:
self.ui.titleEdit.setText('Re: %s' % message.title)
if message.flags & message.IS_HTML:
content = message.content
else:
content = message.content.replace('&', '&').replace('<', '<').replace('>', '>').replace('\n', ' ')
extra = u''
if message.flags & message.IS_NOT_ACCUSED:
extra += u'Status: Unread '
elif message.flags & message.IS_ACCUSED:
extra += u'Status: Read '
elif message.flags & message.IS_UNREAD:
extra += u'Status: New '
self.ui.messageBody.setText("
%s
"
"Date: %s "
"From: %s "
"%s"
"
%s
"
% (message.title, str(message.date), message.sender, extra, content))
if item and message.flags & message.IS_UNREAD:
backend.set_message_read(message)
message.flags &= ~message.IS_UNREAD
item.setForeground(0, QBrush())
item.setForeground(1, QBrush())
item.setForeground(2, QBrush())
def displayReply(self):
self.ui.replyButton.setText(self.tr('Cancel'))
self.ui.replyWidget.show()
def hideReply(self):
self.ui.replyButton.setText(self.tr('Reply'))
self.ui.replyWidget.hide()
self.ui.replyEdit.clear()
self.ui.titleEdit.clear()
def _replyPressed(self):
if self.ui.replyWidget.isVisible():
self.hideReply()
else:
self.displayReply()
def _sendPressed(self):
if not self.ui.replyWidget.isVisible():
return
text = unicode(self.ui.replyEdit.toPlainText())
title = unicode(self.ui.titleEdit.text())
self.ui.backendsList.setEnabled(False)
self.ui.threadsList.setEnabled(False)
self.ui.messagesTree.setEnabled(False)
self.ui.replyButton.setEnabled(False)
self.ui.replyWidget.setEnabled(False)
self.ui.sendButton.setText(self.tr('Sending...'))
flags = 0
if self.ui.htmlBox.currentIndex() == 0:
flags = Message.IS_HTML
m = Message(thread=self.thread,
id=0,
title=title,
sender=None,
receivers=None,
content=text,
parent=self.message,
flags=flags)
self.process_reply = QtDo(self.weboob, self._postReply_cb, self._postReply_eb)
self.process_reply.do('post_message', m, backends=self.thread.backend)
def _postReply_cb(self, backend, ignored):
if not backend:
return
self.ui.backendsList.setEnabled(True)
self.ui.threadsList.setEnabled(True)
self.ui.messagesTree.setEnabled(True)
self.ui.replyButton.setEnabled(True)
self.ui.replyWidget.setEnabled(True)
self.ui.sendButton.setEnabled(True)
self.ui.sendButton.setText(self.tr('Send'))
self.hideReply()
self.process_reply = None
self.refreshThreadMessages(backend.name, self.thread.id)
def _postReply_eb(self, backend, error, backtrace):
content = unicode(self.tr('Unable to send message:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while posting reply'),
content, QMessageBox.Ok)
self.ui.backendsList.setEnabled(True)
self.ui.threadsList.setEnabled(True)
self.ui.messagesTree.setEnabled(True)
self.ui.replyButton.setEnabled(True)
self.ui.replyWidget.setEnabled(True)
self.ui.sendButton.setText(self.tr('Send'))
self.process_reply = None
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qboobmsg/qboobmsg.py 0000664 0000000 0000000 00000002531 11666415431 0030610 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.messages import ICapMessages
from weboob.tools.application.qt import QtApplication
from .main_window import MainWindow
class QBoobMsg(QtApplication):
APPNAME = 'qboobmsg'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
DESCRIPTION = 'Qt application allowing to read messages on various websites and reply to them.'
CAPS = ICapMessages
def main(self, argv):
self.load_backends(ICapMessages, storage=self.create_storage())
self.main_window = MainWindow(self.config, self.weboob)
self.main_window.show()
return self.weboob.loop()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qboobmsg/ui/ 0000775 0000000 0000000 00000000000 11666415431 0027041 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qboobmsg/ui/Makefile 0000664 0000000 0000000 00000000265 11666415431 0030504 0 ustar 00root root 0000000 0000000 UI_FILES = $(wildcard *.ui)
UI_PY_FILES = $(UI_FILES:%.ui=%_ui.py)
PYUIC = pyuic4
all: $(UI_PY_FILES)
%_ui.py: %.ui
$(PYUIC) -o $@ $^
clean:
rm -f *.pyc
rm -f $(UI_PY_FILES)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qboobmsg/ui/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0031140 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qboobmsg/ui/main_window.ui 0000664 0000000 0000000 00000004306 11666415431 0031716 0 ustar 00root root 0000000 0000000
MainWindow00763580QBoobMsg0076320FiletoolBarTopToolBarAreafalseBackendsQuitQuitCtrl+QRefreshactionQuittriggered()MainWindowclose()-1-1381289
messages_manager.ui 0000664 0000000 0000000 00000021005 11666415431 0032620 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qboobmsg/ui
MessagesManager00696591Qt::Horizontal0015016777215Qt::HorizontalQt::Vertical0050QFrame::StyledPanelQFrame::Raised929200+00−Qt::Horizontal4020QAbstractItemView::NoEditTriggerstruetruetruetrue150true150trueTitleFromDateQt::Vertical05trueQt::LinksAccessibleByKeyboard|Qt::LinksAccessibleByMouse|Qt::TextBrowserInteraction|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouseQFrame::StyledPanelQFrame::RaisedReply0With HTMLWithout HTMLSendexpandButtonclicked()messagesTreeexpandAll()73331527150collapseButtonclicked()messagesTreecollapseAll()73360527150
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qhavesex/ 0000775 0000000 0000000 00000000000 11666415431 0026437 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qhavesex/__init__.py 0000664 0000000 0000000 00000000067 11666415431 0030553 0 ustar 00root root 0000000 0000000 from .qhavesex import QHaveSex
__all__ = ['QHaveSex']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qhavesex/contacts.py 0000664 0000000 0000000 00000046044 11666415431 0030637 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import time
import logging
from PyQt4.QtGui import QWidget, QListWidgetItem, QImage, QIcon, QPixmap, \
QFrame, QMessageBox, QTabWidget, QVBoxLayout, \
QFormLayout, QLabel, QPushButton
from PyQt4.QtCore import SIGNAL, Qt
from weboob.tools.application.qt import QtDo, HTMLDelegate
from weboob.tools.misc import to_unicode
from weboob.capabilities.contact import ICapContact, Contact
from weboob.capabilities.chat import ICapChat
from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Message
from weboob.capabilities.base import NotLoaded
from .ui.contacts_ui import Ui_Contacts
from .ui.contact_thread_ui import Ui_ContactThread
from .ui.thread_message_ui import Ui_ThreadMessage
from .ui.profile_ui import Ui_Profile
class ThreadMessage(QFrame):
"""
This class represents a message in the thread tab.
"""
def __init__(self, message, parent=None):
QFrame.__init__(self, parent)
self.ui = Ui_ThreadMessage()
self.ui.setupUi(self)
self.set_message(message)
def set_message(self, message):
self.message = message
self.ui.nameLabel.setText(message.sender)
header = time.strftime('%Y-%m-%d %H:%M:%S', message.date.timetuple())
if message.flags & message.IS_NOT_ACCUSED:
header += u' — Unread'
elif message.flags & message.IS_ACCUSED:
header += u' — Read'
self.ui.headerLabel.setText(header)
if message.flags & message.IS_HTML:
content = message.content
else:
content = message.content.replace('&', '&').replace('<', '<').replace('>', '>').replace('\n', ' ')
self.ui.contentLabel.setText(content)
def __eq__(self, m):
return self.message == m.message
class ContactThread(QWidget):
"""
The thread of the selected contact.
"""
def __init__(self, weboob, contact, support_reply, parent=None):
QWidget.__init__(self, parent)
self.ui = Ui_ContactThread()
self.ui.setupUi(self)
self.weboob = weboob
self.contact = contact
self.thread = None
self.messages = []
self.process_msg = None
self.connect(self.ui.refreshButton, SIGNAL('clicked()'), self.refreshMessages)
if support_reply:
self.connect(self.ui.sendButton, SIGNAL('clicked()'), self.postReply)
else:
self.ui.frame.hide()
self.refreshMessages()
def refreshMessages(self, fillobj=False):
if self.process_msg:
return
self.ui.refreshButton.setEnabled(False)
self.process_msg = QtDo(self.weboob, self.gotThread, self.gotError)
if fillobj and self.thread:
self.process_msg.do('fillobj', self.thread, ['root'], backends=self.contact.backend)
else:
self.process_msg.do('get_thread', self.contact.id, backends=self.contact.backend)
def gotError(self, backend, error, backtrace):
self.ui.textEdit.setEnabled(False)
self.ui.sendButton.setEnabled(False)
self.ui.refreshButton.setEnabled(True)
def gotThread(self, backend, thread):
if not thread:
#v = self.ui.scrollArea.verticalScrollBar()
#print v.minimum(), v.value(), v.maximum(), v.sliderPosition()
#self.ui.scrollArea.verticalScrollBar().setValue(self.ui.scrollArea.verticalScrollBar().maximum())
self.process_msg = None
return
self.ui.textEdit.setEnabled(True)
self.ui.sendButton.setEnabled(True)
self.ui.refreshButton.setEnabled(True)
self.thread = thread
if thread.root is NotLoaded:
self._insert_load_button(0)
else:
for message in thread.iter_all_messages():
self._insert_message(message)
def _insert_message(self, message):
widget = ThreadMessage(message)
if widget in self.messages:
old_widget = self.messages[self.messages.index(widget)]
if old_widget.message.flags != widget.message.flags:
old_widget.set_message(widget.message)
return
for i, m in enumerate(self.messages):
if widget.message.date > m.message.date:
self.ui.scrollAreaContent.layout().insertWidget(i, widget)
self.messages.insert(i, widget)
if message.parent is NotLoaded:
self._insert_load_button(i)
return
self.ui.scrollAreaContent.layout().addWidget(widget)
self.messages.append(widget)
if message.parent is NotLoaded:
self._insert_load_button(-1)
def _insert_load_button(self, pos):
button = QPushButton(self.tr('More messages...'))
self.connect(button, SIGNAL('clicked()'), lambda: self._load_button_pressed(button))
if pos >= 0:
self.ui.scrollAreaContent.layout().insertWidget(pos, button)
else:
self.ui.scrollAreaContent.layout().addWidget(button)
def _load_button_pressed(self, button):
self.ui.scrollAreaContent.layout().removeWidget(button)
button.hide()
button.deleteLater()
self.refreshMessages(fillobj=True)
def postReply(self):
text = unicode(self.ui.textEdit.toPlainText())
self.ui.textEdit.setEnabled(False)
self.ui.sendButton.setEnabled(False)
m = Message(thread=self.thread,
id=0,
title=u'',
sender=None,
receivers=None,
content=text,
parent=self.messages[0].message if len(self.messages) > 0 else None)
self.process_reply = QtDo(self.weboob, self._postReply_cb, self._postReply_eb)
self.process_reply.do('post_message', m, backends=self.contact.backend)
def _postReply_cb(self, backend, ignored):
if not backend:
return
self.ui.textEdit.clear()
self.ui.textEdit.setEnabled(True)
self.ui.sendButton.setEnabled(True)
self.refreshMessages()
self.process_reply = None
def _postReply_eb(self, backend, error, backtrace):
content = unicode(self.tr('Unable to send message:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while posting reply'),
content, QMessageBox.Ok)
self.process_reply = None
class ContactProfile(QWidget):
def __init__(self, weboob, contact, parent=None):
QWidget.__init__(self, parent)
self.ui = Ui_Profile()
self.ui.setupUi(self)
self.connect(self.ui.previousButton, SIGNAL('clicked()'), self.previousClicked)
self.connect(self.ui.nextButton, SIGNAL('clicked()'), self.nextClicked)
self.weboob = weboob
self.contact = contact
self.loaded_profile = False
self.displayed_photo_idx = 0
self.process_photo = {}
missing_fields = self.gotProfile(self.weboob.get_backend(contact.backend), contact)
if len(missing_fields) > 0:
self.process_contact = QtDo(self.weboob, self.gotProfile, self.gotError)
self.process_contact.do('fillobj', self.contact, missing_fields, backends=self.contact.backend)
def gotError(self, backend, error, backtrace):
self.ui.frame_photo.hide()
self.ui.descriptionEdit.setText('
Unable to show profile
%s
' % to_unicode(error))
def gotProfile(self, backend, contact):
if not backend:
return []
missing_fields = set()
self.display_photo()
self.ui.nicknameLabel.setText('
' % contact.summary.replace('\n', ' '))
if not contact.profile:
missing_fields.add('profile')
elif not self.loaded_profile:
self.loaded_profile = True
for head in contact.profile.itervalues():
if head.flags & head.HEAD:
widget = self.ui.headWidget
else:
widget = self.ui.profileTab
self.process_node(head, widget)
return missing_fields
def process_node(self, node, widget):
# Set the value widget
value = None
if node.flags & node.SECTION:
value = QWidget()
value.setLayout(QFormLayout())
for sub in node.value.itervalues():
self.process_node(sub, value)
elif isinstance(node.value, list):
value = QLabel(' '.join(unicode(s) for s in node.value))
value.setWordWrap(True)
elif isinstance(node.value, tuple):
value = QLabel(', '.join(unicode(s) for s in node.value))
value.setWordWrap(True)
elif isinstance(node.value, (basestring,int,long,float)):
value = QLabel(unicode(node.value))
else:
logging.warning('Not supported value: %r' % node.value)
return
# Insert the value widget into the parent widget, depending
# of its type.
if isinstance(widget, QTabWidget):
widget.addTab(value, node.label)
elif isinstance(widget.layout(), QFormLayout):
label = QLabel(u'%s: ' % node.label)
widget.layout().addRow(label, value)
elif isinstance(widget.layout(), QVBoxLayout):
widget.layout().addWidget(QLabel(u'
%s
' % node.label))
widget.layout().addWidget(value)
else:
logging.warning('Not supported widget: %r' % widget)
def previousClicked(self):
self.displayed_photo_idx = (self.displayed_photo_idx - 1) % len(self.contact.photos)
self.display_photo()
def nextClicked(self):
self.displayed_photo_idx = (self.displayed_photo_idx + 1) % len(self.contact.photos)
self.display_photo()
def display_photo(self):
if self.displayed_photo_idx >= len(self.contact.photos):
self.displayed_photo_idx = len(self.contact.photos) - 1
if self.displayed_photo_idx < 0:
self.ui.photoUrlLabel.setText('')
return
photo = self.contact.photos.values()[self.displayed_photo_idx]
if photo.data:
data = photo.data
if photo.id in self.process_photo:
self.process_photo.pop(photo.id)
else:
self.process_photo[photo.id] = QtDo(self.weboob, lambda b,p: self.display_photo())
self.process_photo[photo.id].do('fillobj', photo, ['data'], backends=self.contact.backend)
if photo.thumbnail_data:
data = photo.thumbnail_data
else:
return
img = QImage.fromData(data)
img = img.scaledToWidth(self.width()/3)
self.ui.photoLabel.setPixmap(QPixmap.fromImage(img))
if photo.url is not NotLoaded:
text = '%s' % (photo.url, photo.url)
if photo.hidden:
text += ' (Hidden photo)'
self.ui.photoUrlLabel.setText(text)
class IGroup(object):
def __init__(self, weboob, id, name):
self.id = id
self.name = name
self.weboob = weboob
def iter_contacts(self, cb):
raise NotImplementedError()
class MetaGroup(IGroup):
def iter_contacts(self, cb):
if self.id == 'online':
status = Contact.STATUS_ONLINE|Contact.STATUS_AWAY
elif self.id == 'offline':
status = Contact.STATUS_OFFLINE
else:
status = Contact.STATUS_ALL
self.process = QtDo(self.weboob, lambda b, d: self.cb(cb, b, d))
self.process.do('iter_contacts', status, caps=ICapContact)
def cb(self, cb, backend, contact):
if contact:
cb(contact)
elif not backend:
self.process = None
cb(None)
class ContactsWidget(QWidget):
def __init__(self, weboob, parent=None):
QWidget.__init__(self, parent)
self.ui = Ui_Contacts()
self.ui.setupUi(self)
self.weboob = weboob
self.contact = None
self.ui.contactList.setItemDelegate(HTMLDelegate())
self.url_process = None
self.photo_processes = {}
self.ui.groupBox.addItem('All', MetaGroup(self.weboob, 'all', self.tr('All')))
self.ui.groupBox.addItem('Onlines', MetaGroup(self.weboob, 'online', self.tr('Online')))
self.ui.groupBox.addItem('Offlines', MetaGroup(self.weboob, 'offline', self.tr('Offline')))
self.ui.groupBox.setCurrentIndex(1)
self.connect(self.ui.groupBox, SIGNAL('currentIndexChanged(int)'), self.groupChanged)
self.connect(self.ui.contactList, SIGNAL('itemClicked(QListWidgetItem*)'), self.contactChanged)
self.connect(self.ui.refreshButton, SIGNAL('clicked()'), self.refreshContactList)
self.connect(self.ui.urlButton, SIGNAL('clicked()'), self.urlClicked)
def load(self):
self.refreshContactList()
self.ui.backendsList.clear()
for backend in self.weboob.iter_backends():
self.ui.backendsList.addItem(backend.name)
def groupChanged(self, i):
self.refreshContactList()
def refreshContactList(self):
self.ui.contactList.clear()
self.ui.refreshButton.setEnabled(False)
i = self.ui.groupBox.currentIndex()
group = self.ui.groupBox.itemData(i).toPyObject()
group.iter_contacts(self.addContact)
def setPhoto(self, contact, item):
if not contact:
return False
try:
self.photo_processes.pop(contact.id, None)
except KeyError:
pass
img = None
for photo in contact.photos.itervalues():
if photo.thumbnail_data:
img = QImage.fromData(photo.thumbnail_data)
break
if img:
item.setIcon(QIcon(QPixmap.fromImage(img)))
return True
return False
def addContact(self, contact):
if not contact:
self.ui.refreshButton.setEnabled(True)
return
status = ''
if contact.status == Contact.STATUS_ONLINE:
status = u'Online'
status_color = 0x00aa00
elif contact.status == Contact.STATUS_OFFLINE:
status = u'Offline'
status_color = 0xff0000
elif contact.status == Contact.STATUS_AWAY:
status = u'Away'
status_color = 0xffad16
else:
status = u'Unknown'
status_color = 0xaaaaaa
if contact.status_msg:
status += u' — %s' % contact.status_msg
item = QListWidgetItem()
item.setText('
%s
%s %s' % (contact.name, status_color, status, contact.backend))
item.setData(Qt.UserRole, contact)
if contact.photos is NotLoaded:
process = QtDo(self.weboob, lambda b, c: self.setPhoto(c, item))
process.do('fillobj', contact, ['photos'], backends=contact.backend)
self.photo_processes[contact.id] = process
elif len(contact.photos) > 0:
if not self.setPhoto(contact, item):
photo = contact.photos.values()[0]
process = QtDo(self.weboob, lambda b, p: self.setPhoto(contact, item))
process.do('fillobj', photo, ['thumbnail_data'], backends=contact.backend)
self.photo_processes[contact.id] = process
for i in xrange(self.ui.contactList.count()):
if self.ui.contactList.item(i).data(Qt.UserRole).toPyObject().status > contact.status:
self.ui.contactList.insertItem(i, item)
return
self.ui.contactList.addItem(item)
def contactChanged(self, current):
if not current:
return
contact = current.data(Qt.UserRole).toPyObject()
self.setContact(contact)
def setContact(self, contact):
if not contact or contact == self.contact:
return
self.ui.tabWidget.clear()
self.contact = contact
backend = self.weboob.get_backend(self.contact.backend)
self.ui.tabWidget.addTab(ContactProfile(self.weboob, self.contact), self.tr('Profile'))
if backend.has_caps(ICapMessages):
self.ui.tabWidget.addTab(ContactThread(self.weboob, self.contact, backend.has_caps(ICapMessagesPost)), self.tr('Messages'))
if backend.has_caps(ICapChat):
self.ui.tabWidget.addTab(QWidget(), self.tr('Chat'))
self.ui.tabWidget.addTab(QWidget(), self.tr('Calendar'))
self.ui.tabWidget.addTab(QWidget(), self.tr('Notes'))
def urlClicked(self):
url = unicode(self.ui.urlEdit.text())
if not url:
return
backend_name = unicode(self.ui.backendsList.currentText())
self.ui.urlButton.setEnabled(False)
self.url_process = QtDo(self.weboob, self.urlClicked_cb, self.urlClicked_eb)
self.url_process.do('get_contact', url, backends=backend_name)
def urlClicked_cb(self, backend, contact):
if not backend:
self.url_process = None
self.ui.urlButton.setEnabled(True)
return
self.ui.urlEdit.clear()
self.setContact(contact)
def urlClicked_eb(self, backend, error, backtrace):
content = unicode(self.tr('Unable to get contact:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += u'\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while getting contact'),
content, QMessageBox.Ok)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qhavesex/main_window.py 0000664 0000000 0000000 00000005156 11666415431 0031333 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from PyQt4.QtGui import QWidget
from PyQt4.QtCore import SIGNAL
from weboob.tools.application.qt import QtMainWindow
from weboob.tools.application.qt.backendcfg import BackendCfg
from weboob.capabilities.dating import ICapDating
try:
from weboob.applications.qboobmsg.messages_manager import MessagesManager
HAVE_BOOBMSG = True
except ImportError:
HAVE_BOOBMSG = False
from .ui.main_window_ui import Ui_MainWindow
from .status import AccountsStatus
from .contacts import ContactsWidget
class MainWindow(QtMainWindow):
def __init__(self, config, weboob, parent=None):
QtMainWindow.__init__(self, parent)
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
self.config = config
self.weboob = weboob
self.loaded_tabs = {}
self.connect(self.ui.actionBackends, SIGNAL("triggered()"), self.backendsConfig)
self.connect(self.ui.tabWidget, SIGNAL('currentChanged(int)'), self.tabChanged)
self.ui.tabWidget.addTab(AccountsStatus(self.weboob), self.tr('Status'))
if HAVE_BOOBMSG:
self.ui.tabWidget.addTab(MessagesManager(self.weboob), self.tr('Messages'))
self.ui.tabWidget.addTab(ContactsWidget(self.weboob), self.tr('Contacts'))
self.ui.tabWidget.addTab(QWidget(), self.tr('Calendar'))
self.ui.tabWidget.addTab(QWidget(), self.tr('Optimizations'))
if self.weboob.count_backends() == 0:
self.backendsConfig()
def backendsConfig(self):
bckndcfg = BackendCfg(self.weboob, (ICapDating,), self)
if bckndcfg.run():
self.loaded_tabs.clear()
widget = self.ui.tabWidget.widget(self.ui.tabWidget.currentIndex())
widget.load()
def tabChanged(self, i):
widget = self.ui.tabWidget.currentWidget()
if hasattr(widget, 'load') and not i in self.loaded_tabs:
widget.load()
self.loaded_tabs[i] = True
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qhavesex/qhavesex.py 0000664 0000000 0000000 00000002577 11666415431 0030650 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.dating import ICapDating
from weboob.tools.application.qt import QtApplication
from .main_window import MainWindow
class QHaveSex(QtApplication):
APPNAME = 'qhavesex'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
DESCRIPTION = 'Qt application allowing to interact with various dating websites.'
CAPS = ICapDating
STORAGE_FILENAME = 'dating.storage'
def main(self, argv):
self.create_storage(self.STORAGE_FILENAME)
self.load_backends(ICapDating)
self.main_window = MainWindow(self.config, self.weboob)
self.main_window.show()
return self.weboob.loop()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qhavesex/status.py 0000664 0000000 0000000 00000010527 11666415431 0030341 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from PyQt4.QtGui import QScrollArea, QWidget, QHBoxLayout, QVBoxLayout, QFrame, QLabel, QImage, QPixmap
from weboob.capabilities.account import ICapAccount, StatusField
from weboob.tools.application.qt import QtDo
from weboob.tools.misc import to_unicode
class Account(QFrame):
def __init__(self, weboob, backend, parent=None):
QFrame.__init__(self, parent)
self.setFrameShape(QFrame.StyledPanel)
self.setFrameShadow(QFrame.Raised)
self.weboob = weboob
self.backend = backend
self.setLayout(QVBoxLayout())
self.timer = None
head = QHBoxLayout()
headw = QWidget()
headw.setLayout(head)
self.title = QLabel(u'
%s — %s
' % (backend.name, backend.DESCRIPTION))
self.body = QLabel()
if backend.ICON:
self.icon = QLabel()
img = QImage(backend.ICON)
self.icon.setPixmap(QPixmap.fromImage(img))
head.addWidget(self.icon)
head.addWidget(self.title)
head.addStretch()
self.layout().addWidget(headw)
if backend.has_caps(ICapAccount):
self.body.setText(u'Waiting...')
self.layout().addWidget(self.body)
self.timer = self.weboob.repeat(60, self.updateStats)
def deinit(self):
if self.timer is not None:
self.weboob.stop(self.timer)
def updateStats(self):
self.process = QtDo(self.weboob, self.updateStats_cb, self.updateStats_eb)
self.process.body = u''
self.process.in_p = False
self.process.do('get_account_status', backends=self.backend)
def updateStats_cb(self, backend, field):
if not field:
if self.process.in_p:
self.process.body += u"
"
self.body.setText(self.process.body)
self.process = None
return
if field.flags & StatusField.FIELD_HTML:
value = u'%s' % field.value
else:
value = (u'%s' % field.value).replace('&', '&').replace('<', '<').replace('>', '>')
if field.flags & StatusField.FIELD_TEXT:
if self.process.in_p:
self.process.body += u''
self.process.body += u'
%s
' % value
self.process.in_p = False
else:
if not self.process.in_p:
self.process.body += u"
"
self.process.in_p = True
else:
self.process.body += u" "
self.process.body += u'%s: %s' % (field.label, field.value)
def updateStats_eb(self, backend, err, backtrace):
self.body.setText(u'Unable to connect: %s' % to_unicode(err))
self.title.setText(u'%s' % unicode(self.title.text()))
class AccountsStatus(QScrollArea):
def __init__(self, weboob, parent=None):
QScrollArea.__init__(self, parent)
self.weboob = weboob
self.setFrameShadow(self.Plain)
self.setFrameShape(self.NoFrame)
self.setWidgetResizable(True)
widget = QWidget(self)
widget.setLayout(QVBoxLayout())
widget.show()
self.setWidget(widget)
def load(self):
while self.widget().layout().count() > 0:
item = self.widget().layout().takeAt(0)
if item.widget():
item.widget().deinit()
item.widget().hide()
item.widget().deleteLater()
for backend in self.weboob.iter_backends():
account = Account(self.weboob, backend)
self.widget().layout().addWidget(account)
self.widget().layout().addStretch()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qhavesex/ui/ 0000775 0000000 0000000 00000000000 11666415431 0027054 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qhavesex/ui/Makefile 0000664 0000000 0000000 00000000265 11666415431 0030517 0 ustar 00root root 0000000 0000000 UI_FILES = $(wildcard *.ui)
UI_PY_FILES = $(UI_FILES:%.ui=%_ui.py)
PYUIC = pyuic4
all: $(UI_PY_FILES)
%_ui.py: %.ui
$(PYUIC) -o $@ $^
clean:
rm -f *.pyc
rm -f $(UI_PY_FILES)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qhavesex/ui/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0031153 0 ustar 00root root 0000000 0000000 contact_thread.ui 0000664 0000000 0000000 00000011121 11666415431 0032312 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qhavesex/ui
ContactThread00578429FormQt::Vertical00QFrame::StyledPanelQFrame::Raisedfalsefalse00Send050Qt::Horizontal4020../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../usr/share/icons/oxygen/16x16/actions/view-refresh.png../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../usr/share/icons/oxygen/16x16/actions/view-refresh.png01Qt::ScrollBarAsNeededtrueQt::AlignBottom|Qt::AlignLeading|Qt::AlignLeft00556154QWidget#scrollAreaContent {
background-color: rgb(255, 255, 255);
}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qhavesex/ui/contacts.ui 0000664 0000000 0000000 00000007306 11666415431 0031237 0 ustar 00root root 0000000 0000000
Contacts00478374FormQt::Horizontal00QFrame::StyledPanelQFrame::Raised../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../usr/share/icons/oxygen/16x16/actions/view-refresh.png../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../usr/share/icons/oxygen/16x16/actions/view-refresh.png1201201truefalsetrueFrom URLDisplay10
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qhavesex/ui/main_window.ui 0000664 0000000 0000000 00000004166 11666415431 0031735 0 ustar 00root root 0000000 0000000
MainWindow00763580QHaveSex-10076324FiletoolBarTopToolBarAreafalseBackendsQuitQuitactionQuittriggered()MainWindowclose()-1-1381289
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qhavesex/ui/profile.ui 0000664 0000000 0000000 00000020655 11666415431 0031063 0 ustar 00root root 0000000 0000000
Profile00755647Form0QFrame::NoFrameQFrame::Plaintrue0075564700000QFrame::StyledPanelQFrame::Raised<h1>Loading...</h1><b>URL:</b>trueQt::LinksAccessibleByMouse|Qt::TextSelectableByMouse0900Qt::Vertical200QFrame::StyledPanelQFrame::Raised002016777215<Qt::AlignCenter002016777215>Qt::AlignCenterQt::LinksAccessibleByMouse|Qt::TextSelectableByMouseQt::Vertical130Qt::Horizontal00true20-1
thread_message.ui 0000664 0000000 0000000 00000005367 11666415431 0032322 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qhavesex/ui
ThreadMessage0055276FrameQFrame::StyledPanelQFrame::RaisedQt::Vertical20100trueQt::LinksAccessibleByMouse|Qt::TextSelectableByMousetrueQt::LinksAccessibleByMouse|Qt::TextSelectableByMouseQt::Vertical201
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qvideoob/ 0000775 0000000 0000000 00000000000 11666415431 0026423 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qvideoob/__init__.py 0000664 0000000 0000000 00000000067 11666415431 0030537 0 ustar 00root root 0000000 0000000 from .qvideoob import QVideoob
__all__ = ['QVideoob']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qvideoob/main_window.py 0000664 0000000 0000000 00000012421 11666415431 0031310 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from PyQt4.QtCore import SIGNAL
from weboob.capabilities.video import ICapVideo
from weboob.tools.application.qt import QtMainWindow, QtDo
from weboob.tools.application.qt.backendcfg import BackendCfg
from weboob.applications.qvideoob.ui.main_window_ui import Ui_MainWindow
from .video import Video
from .minivideo import MiniVideo
class MainWindow(QtMainWindow):
def __init__(self, config, weboob, parent=None):
QtMainWindow.__init__(self, parent)
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
self.config = config
self.weboob = weboob
self.minivideos = []
self.ui.sortbyEdit.setCurrentIndex(int(self.config.get('settings', 'sortby')))
self.ui.nsfwCheckBox.setChecked(int(self.config.get('settings', 'nsfw')))
self.ui.sfwCheckBox.setChecked(int(self.config.get('settings', 'sfw')))
self.connect(self.ui.searchEdit, SIGNAL("returnPressed()"), self.search)
self.connect(self.ui.urlEdit, SIGNAL("returnPressed()"), self.openURL)
self.connect(self.ui.nsfwCheckBox, SIGNAL("stateChanged(int)"), self.nsfwChanged)
self.connect(self.ui.sfwCheckBox, SIGNAL("stateChanged(int)"), self.sfwChanged)
self.connect(self.ui.actionBackends, SIGNAL("triggered()"), self.backendsConfig)
self.loadBackendsList()
if self.ui.backendEdit.count() == 0:
self.backendsConfig()
def backendsConfig(self):
bckndcfg = BackendCfg(self.weboob, (ICapVideo,), self)
if bckndcfg.run():
self.loadBackendsList()
def loadBackendsList(self):
self.ui.backendEdit.clear()
for i, backend in enumerate(self.weboob.iter_backends()):
if i == 0:
self.ui.backendEdit.addItem('All backends', '')
self.ui.backendEdit.addItem(backend.name, backend.name)
if backend.name == self.config.get('settings', 'backend'):
self.ui.backendEdit.setCurrentIndex(i+1)
if self.ui.backendEdit.count() == 0:
self.ui.searchEdit.setEnabled(False)
self.ui.urlEdit.setEnabled(False)
else:
self.ui.searchEdit.setEnabled(True)
self.ui.urlEdit.setEnabled(True)
def nsfwChanged(self, state):
self.config.set('settings', 'nsfw', int(self.ui.nsfwCheckBox.isChecked()))
self.updateVideosDisplay()
def sfwChanged(self, state):
self.config.set('settings', 'sfw', int(self.ui.sfwCheckBox.isChecked()))
self.updateVideosDisplay()
def updateVideosDisplay(self):
for minivideo in self.minivideos:
if (minivideo.video.nsfw and self.ui.nsfwCheckBox.isChecked() or
not minivideo.video.nsfw and self.ui.sfwCheckBox.isChecked()):
minivideo.show()
else:
minivideo.hide()
def search(self):
pattern = unicode(self.ui.searchEdit.text())
if not pattern:
return
for minivideo in self.minivideos:
self.ui.scrollAreaContent.layout().removeWidget(minivideo)
minivideo.hide()
minivideo.deleteLater()
self.minivideos = []
self.ui.searchEdit.setEnabled(False)
backend_name = str(self.ui.backendEdit.itemData(self.ui.backendEdit.currentIndex()).toString())
self.process = QtDo(self.weboob, self.addVideo)
self.process.do('iter_search_results', pattern, self.ui.sortbyEdit.currentIndex(), nsfw=True, max_results=20, backends=backend_name)
def addVideo(self, backend, video):
if not backend:
self.ui.searchEdit.setEnabled(True)
self.process = None
return
minivideo = MiniVideo(self.weboob, backend, video)
self.ui.scrollAreaContent.layout().addWidget(minivideo)
self.minivideos.append(minivideo)
if (video.nsfw and not self.ui.nsfwCheckBox.isChecked() or
not video.nsfw and not self.ui.sfwCheckBox.isChecked()):
minivideo.hide()
def openURL(self):
url = unicode(self.ui.urlEdit.text())
if not url:
return
for backend in self.weboob.iter_backends():
video = backend.get_video(url)
if video:
video_widget = Video(video, self)
video_widget.show()
self.ui.urlEdit.clear()
def closeEvent(self, ev):
self.config.set('settings', 'backend', str(self.ui.backendEdit.itemData(self.ui.backendEdit.currentIndex()).toString()))
self.config.set('settings', 'sortby', self.ui.sortbyEdit.currentIndex())
self.config.save()
ev.accept()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qvideoob/minivideo.py 0000664 0000000 0000000 00000005016 11666415431 0030762 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from PyQt4.QtGui import QFrame, QImage, QPixmap
from weboob.tools.application.qt import QtDo
from weboob.applications.qvideoob.ui.minivideo_ui import Ui_MiniVideo
from .video import Video
class MiniVideo(QFrame):
def __init__(self, weboob, backend, video, parent=None):
QFrame.__init__(self, parent)
self.ui = Ui_MiniVideo()
self.ui.setupUi(self)
self.weboob = weboob
self.backend = backend
self.video = video
self.ui.titleLabel.setText(video.title)
self.ui.backendLabel.setText(backend.name)
self.ui.durationLabel.setText(unicode(video.duration))
self.ui.authorLabel.setText(unicode(video.author))
self.ui.dateLabel.setText(video.date and unicode(video.date) or '')
if video.rating_max:
self.ui.ratingLabel.setText('%s / %s' % (video.rating, video.rating_max))
else:
self.ui.ratingLabel.setText('%s' % video.rating)
self.process_thumbnail = QtDo(self.weboob, self.gotThumbnail)
self.process_thumbnail.do('fillobj', self.video, ['thumbnail'], backends=backend)
def gotThumbnail(self, backend, video):
if not backend:
return
if video.thumbnail and video.thumbnail.data:
img = QImage.fromData(video.thumbnail.data)
self.ui.imageLabel.setPixmap(QPixmap.fromImage(img))
def enterEvent(self, event):
self.setFrameShadow(self.Sunken)
QFrame.enterEvent(self, event)
def leaveEvent(self, event):
self.setFrameShadow(self.Raised)
QFrame.leaveEvent(self, event)
def mousePressEvent(self, event):
QFrame.mousePressEvent(self, event)
video = self.backend.get_video(self.video.id)
if video:
video_widget = Video(video, self)
video_widget.show()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qvideoob/qvideoob.py 0000664 0000000 0000000 00000003013 11666415431 0030602 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.video import ICapVideo
from weboob.tools.application.qt import QtApplication
from .main_window import MainWindow
class QVideoob(QtApplication):
APPNAME = 'qvideoob'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
DESCRIPTION = 'Qt application allowing to search videos on various websites and play them.'
CAPS = ICapVideo
CONFIG = {'settings': {'nsfw': 1,
'sfw': 1,
'sortby': 0,
'backend': ''
}
}
def main(self, argv):
self.load_backends(ICapVideo)
self.load_config()
self.main_window = MainWindow(self.config, self.weboob)
self.main_window.show()
return self.weboob.loop()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qvideoob/ui/ 0000775 0000000 0000000 00000000000 11666415431 0027040 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qvideoob/ui/Makefile 0000664 0000000 0000000 00000000265 11666415431 0030503 0 ustar 00root root 0000000 0000000 UI_FILES = $(wildcard *.ui)
UI_PY_FILES = $(UI_FILES:%.ui=%_ui.py)
PYUIC = pyuic4
all: $(UI_PY_FILES)
%_ui.py: %.ui
$(PYUIC) -o $@ $^
clean:
rm -f *.pyc
rm -f $(UI_PY_FILES)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qvideoob/ui/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0031137 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qvideoob/ui/main_window.ui 0000664 0000000 0000000 00000012651 11666415431 0031717 0 ustar 00root root 0000000 0000000
MainWindow00582463QVideoobQFrame::StyledPanelQFrame::RaisedSearch: 00RelevanceRatingDurationDate10000Display:SFWtrueNSFWtrueQt::Horizontal4020true00560230QWidget#scrollAreaContent {
background-color: rgb(255, 255, 255);
}QFrame::StyledPanelQFrame::RaisedURL: 0058225toolBarTopToolBarAreafalseBackends
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qvideoob/ui/minivideo.ui 0000664 0000000 0000000 00000011337 11666415431 0031367 0 ustar 00root root 0000000 0000000
MiniVideo00464132FormQFrame::StyledPanelQFrame::Raised95500QFormLayout::ExpandingFieldsGrow75trueTitle0050truefalseTextLabel75trueDurationTextLabel75trueAuthorTextLabel75trueDateTextLabel75trueRatingTextLabel75trueWhereTextLabel
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qvideoob/ui/video.ui 0000664 0000000 0000000 00000013711 11666415431 0030510 0 ustar 00root root 0000000 0000000
Video00647404Video1275trueQFrame::BoxQFrame::RaisedTextLabelQt::AlignCenter00background-color: rgb(255, 255, 255);QFrame::StyledPanelQFrame::Sunken00truetrueQFrame::StyledPanelQFrame::RaisedQFormLayout::ExpandingFieldsGrow75trueURLtrueArrowCursortruefalsetrue75trueDurationTextLabel75trueAuthorTextLabel75trueDateTextLabel75trueRatingTextLabelPhonon::VideoPlayerQWidgetphonon/videoplayer.hPhonon::SeekSliderQWidgetphonon/seekslider.h
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qvideoob/video.py 0000664 0000000 0000000 00000003524 11666415431 0030107 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from PyQt4.QtCore import QUrl
from PyQt4.QtGui import QDialog
from PyQt4.phonon import Phonon
from weboob.applications.qvideoob.ui.video_ui import Ui_Video
class Video(QDialog):
def __init__(self, video, parent=None):
QDialog.__init__(self, parent)
self.ui = Ui_Video()
self.ui.setupUi(self)
self.video = video
self.setWindowTitle("Video - %s" % video.title)
self.ui.urlEdit.setText(video.url)
self.ui.titleLabel.setText(video.title)
self.ui.durationLabel.setText(unicode(video.duration))
self.ui.authorLabel.setText(unicode(video.author))
self.ui.dateLabel.setText(unicode(video.date))
if video.rating_max:
self.ui.ratingLabel.setText('%s / %s' % (video.rating, video.rating_max))
else:
self.ui.ratingLabel.setText('%s' % video.rating)
self.ui.seekSlider.setMediaObject(self.ui.videoPlayer.mediaObject())
self.ui.videoPlayer.load(Phonon.MediaSource(QUrl(video.url)))
self.ui.videoPlayer.play()
def closeEvent(self, event):
self.ui.videoPlayer.stop()
event.accept()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qwebcontentedit/ 0000775 0000000 0000000 00000000000 11666415431 0030012 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qwebcontentedit/__init__.py0000664 0000000 0000000 00000000113 11666415431 0032116 0 ustar 00root root 0000000 0000000 from .qwebcontentedit import QWebContentEdit
__all__ = ['QWebContentEdit'] main_window.py 0000664 0000000 0000000 00000012467 11666415431 0032632 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qwebcontentedit # -*- coding: utf-8 -*-
# Copyright(C) 2011 Clément Schreiner
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import logging
from copy import deepcopy
from PyQt4.QtCore import SIGNAL
from PyQt4.QtGui import QMessageBox
from weboob.tools.application.qt import QtMainWindow, QtDo
from weboob.tools.application.qt.backendcfg import BackendCfg
from weboob.capabilities.content import ICapContent
from weboob.tools.misc import to_unicode
from .ui.main_window_ui import Ui_MainWindow
class MainWindow(QtMainWindow):
def __init__(self, config, weboob, parent=None):
QtMainWindow.__init__(self, parent)
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
self.config = config
self.weboob = weboob
self.backend = None
self.connect(self.ui.idEdit, SIGNAL("returnPressed()"), self.loadPage)
self.connect(self.ui.loadButton, SIGNAL("clicked()"), self.loadPage)
self.connect(self.ui.tabWidget, SIGNAL("currentChanged(int)"),
self._currentTabChanged)
self.connect(self.ui.saveButton, SIGNAL("clicked()"), self.savePage)
self.connect(self.ui.actionBackends, SIGNAL("triggered()"), self.backendsConfig)
if self.weboob.count_backends() == 0:
self.backendsConfig()
else:
self.loadBackends()
def backendsConfig(self):
bckndcfg = BackendCfg(self.weboob, (ICapContent,), self)
if bckndcfg.run():
self.loadBackends()
def loadBackends(self):
self.ui.backendBox.clear()
for backend in self.weboob.iter_backends():
self.ui.backendBox.insertItem(0, backend.name)
def _currentTabChanged(self):
if self.ui.tabWidget.currentIndex() == 1:
if self.backend is not None:
self.showPreview()
def loadPage(self):
_id = unicode(self.ui.idEdit.text())
if not _id:
return
self.ui.saveButton.setEnabled(False)
backend = str(self.ui.backendBox.currentText())
self.process = QtDo(self.weboob, self._loadPage_cb, self._loadPage_eb)
self.process.do('get_content', _id, backends=(backend,))
def _loadPage_cb(self, backend, data):
if not backend:
self.process = None
if self.backend:
self.ui.saveButton.setEnabled(True)
return
if not data:
self.content = None
self.backend = None
QMessageBox.critical(self, self.tr('Unable to open page'),
'Unable to open page "%s" on %s: it does not exist.'
% (self.ui.idEdit.text(), self.ui.backendBox.currentText()),
QMessageBox.Ok)
return
self.content = data
self.ui.contentEdit.setPlainText(self.content.content)
self.setWindowTitle("QWebcontentedit - %s@%s" %(self.content.id, backend.name))
self.backend = backend
def _loadPage_eb(self, backend, error, backtrace):
content = unicode(self.tr('Unable to load page:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while loading page'),
content, QMessageBox.Ok)
def savePage(self):
if self.backend is None:
return
new_content = unicode(self.ui.contentEdit.toPlainText())
minor = self.ui.minorBox.isChecked()
if new_content != self.content.content:
self.ui.saveButton.setEnabled(False)
self.content.content = new_content
message = unicode(self.ui.descriptionEdit.text())
self.process = QtDo(self.weboob, self._savePage_cb, self._savePage_eb)
self.process.do('push_content', self.content, message, minor=minor, backends=self.backend)
def _savePage_cb(self, backend, data):
if not backend:
self.process = None
self.ui.saveButton.setEnabled(True)
return
self.ui.descriptionEdit.clear()
def _savePage_eb(self, backend, error, backtrace):
content = unicode(self.tr('Unable to save page:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while saving page'),
content, QMessageBox.Ok)
self.ui.saveButton.setEnabled(True)
def showPreview(self):
tmp_content = deepcopy(self.content)
tmp_content.content=unicode(self.ui.contentEdit.toPlainText())
self.ui.previewEdit.setHtml(self.backend.get_content_preview(tmp_content))
qwebcontentedit.py 0000664 0000000 0000000 00000002525 11666415431 0033510 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qwebcontentedit # -*- coding: utf-8 -*-
# Copyright(C) 2011 Clément Schreiner
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.application.qt import QtApplication
from weboob.capabilities.content import ICapContent
from .main_window import MainWindow
class QWebContentEdit(QtApplication):
APPNAME = 'qwebcontentedit'
VERSION = '0.9.1'
COPYRIGHT = u'Copyright(C) 2011 Clément Schreiner'
DESCRIPTION = 'Qt application allowing to manage contents of various websites.'
CAPS = ICapContent
def main(self, argv):
self.load_backends(ICapContent, storage=self.create_storage())
self.main_window = MainWindow(self.config, self.weboob)
self.main_window.show()
return self.weboob.loop()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qwebcontentedit/ui/ 0000775 0000000 0000000 00000000000 11666415431 0030427 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qwebcontentedit/ui/Makefile0000664 0000000 0000000 00000000265 11666415431 0032072 0 ustar 00root root 0000000 0000000 UI_FILES = $(wildcard *.ui)
UI_PY_FILES = $(UI_FILES:%.ui=%_ui.py)
PYUIC = pyuic4
all: $(UI_PY_FILES)
%_ui.py: %.ui
$(PYUIC) -o $@ $^
clean:
rm -f *.pyc
rm -f $(UI_PY_FILES)
__init__.py 0000664 0000000 0000000 00000000000 11666415431 0032447 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qwebcontentedit/ui main_window.ui 0000664 0000000 0000000 00000010000 11666415431 0033211 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qwebcontentedit/ui
MainWindow00469520QWebcontenteditfalseLoad0EditPreviewtrueEdit descriptionMinor editfalseSave0046920FiletoolBarTopToolBarAreafalseExitBackendsactionExittriggered()MainWindowclose()-1-1343306
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qweboobcfg/ 0000775 0000000 0000000 00000000000 11666415431 0026731 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qweboobcfg/__init__.py 0000664 0000000 0000000 00000001435 11666415431 0031045 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .qweboobcfg import QWeboobCfg
__all__ = ['QWeboobCfg']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/qweboobcfg/qweboobcfg.py 0000664 0000000 0000000 00000002500 11666415431 0031416 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ft=python et softtabstop=4 cinoptions=4 shiftwidth=4 ts=4 ai
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.application.qt import BackendCfg, QtApplication
class QWeboobCfg(QtApplication):
APPNAME = 'qweboobcfg'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
DESCRIPTION = "weboob-config-qt is a graphical application to add/edit/remove backends, " \
"and to register new website accounts."
def main(self, argv):
self.load_backends()
self.dlg = BackendCfg(self.weboob)
self.dlg.show()
return self.weboob.loop()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/radioob/ 0000775 0000000 0000000 00000000000 11666415431 0026232 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/radioob/__init__.py 0000664 0000000 0000000 00000001424 11666415431 0030344 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .radioob import Radioob
__all__ = ['Radioob']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/radioob/radioob.py 0000664 0000000 0000000 00000011465 11666415431 0030232 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import sys
from weboob.capabilities.radio import ICapRadio
from weboob.capabilities.base import NotLoaded
from weboob.tools.application.repl import ReplApplication
from weboob.tools.application.media_player import InvalidMediaPlayer, MediaPlayer, MediaPlayerNotFound
from weboob.tools.application.formatters.iformatter import IFormatter
__all__ = ['Radioob']
class RadioListFormatter(IFormatter):
MANDATORY_FIELDS = ('id', 'title', 'description')
count = 0
def flush(self):
self.count = 0
pass
def format_dict(self, item):
self.count += 1
if self.interactive:
backend = item['id'].split('@', 1)[1]
result = u'%s* (%d) %s (%s)%s\n' % (ReplApplication.BOLD, self.count, item['title'], backend, ReplApplication.NC)
else:
result = u'%s* (%s) %s%s\n' % (ReplApplication.BOLD, item['id'], item['title'], ReplApplication.NC)
result += ' %-30s' % item['description']
if item['current'] is not NotLoaded:
if item['current'].artist:
result += ' (Current: %s - %s)' % (item['current'].artist, item['current'].title)
else:
result += ' (Current: %s)' % item['current'].title
return result
class Radioob(ReplApplication):
APPNAME = 'radioob'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
DESCRIPTION = 'Console application allowing to search for web radio stations, listen to them and get information ' \
'like the current song.'
CAPS = ICapRadio
EXTRA_FORMATTERS = {'radio_list': RadioListFormatter}
COMMANDS_FORMATTERS = {'ls': 'radio_list',
'search': 'radio_list',
}
def __init__(self, *args, **kwargs):
ReplApplication.__init__(self, *args, **kwargs)
self.player = MediaPlayer(self.logger)
def main(self, argv):
self.load_config()
return ReplApplication.main(self, argv)
def complete_play(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_object()
def do_play(self, _id):
"""
play ID
Play a radio with a found player.
"""
if not _id:
print >>sys.stderr, 'This command takes an argument: %s' % self.get_command_help('play', short=True)
return 2
radio = self.get_object(_id, 'get_radio', ['streams'])
if not radio:
print >>sys.stderr, 'Radio not found: ' % _id
return 1
try:
player_name = self.config.get('media_player')
if not player_name:
self.logger.debug(u'You can set the media_player key to the player you prefer in the radioob '
'configuration file.')
self.player.play(radio.streams[0], player_name=player_name)
except (InvalidMediaPlayer, MediaPlayerNotFound), e:
print '%s\nRadio URL: %s' % (e, radio.streams[0].url)
def complete_info(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_object()
def do_info(self, _id):
"""
info ID
Get information about a radio.
"""
if not _id:
print >>sys.stderr, 'This command takes an argument: %s' % self.get_command_help('info', short=True)
return 2
radio = self.get_object(_id, 'get_radio')
if not radio:
print >>sys.stderr, 'Radio not found:', _id
return 3
self.format(radio)
self.flush()
def do_search(self, pattern=None):
"""
search PATTERN
List radios matching a PATTERN.
If PATTERN is not given, this command will list all the radios.
"""
self.set_formatter_header(u'Search pattern: %s' % pattern if pattern else u'All radios')
self.change_path('/search')
for backend, radio in self.do('iter_radios_search', pattern=pattern):
self.add_object(radio)
self.format(radio)
self.flush()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/traveloob/ 0000775 0000000 0000000 00000000000 11666415431 0026610 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/traveloob/__init__.py 0000664 0000000 0000000 00000001536 11666415431 0030726 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# vim: ft=python et softtabstop=4 cinoptions=4 shiftwidth=4 ts=4 ai
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .traveloob import Traveloob
__all__ = ['Traveloob']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/traveloob/traveloob.py 0000664 0000000 0000000 00000007776 11666415431 0031200 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Julien Hébert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import sys
from datetime import datetime
import logging
from weboob.capabilities.travel import ICapTravel, RoadmapFilters
from weboob.tools.application.repl import ReplApplication
__all__ = ['Traveloob']
class Traveloob(ReplApplication):
APPNAME = 'traveloob'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
DESCRIPTION = 'Console application allowing to search for train stations and get departure times.'
CAPS = ICapTravel
DEFAULT_FORMATTER = 'table'
def add_application_options(self, group):
group.add_option('--departure-time')
group.add_option('--arrival-time')
def do_stations(self, pattern):
"""
stations PATTERN
Search stations.
"""
for backend, station in self.do('iter_station_search', pattern):
self.format(station)
self.flush()
def do_departures(self, line):
"""
departures STATION [ARRIVAL]
List all departures for a given station.
"""
station, arrival = self.parse_command_args(line, 2, 1)
station_id, backend_name = self.parse_id(station)
if arrival:
arrival_id, backend_name2 = self.parse_id(arrival)
if backend_name and backend_name2 and backend_name != backend_name2:
logging.error('Departure and arrival aren\'t on the same backend')
return 1
else:
arrival_id = backend_name2 = None
if backend_name:
backends = [backend_name]
elif backend_name2:
backends = [backend_name2]
else:
backends = None
for backend, departure in self.do('iter_station_departures', station_id, arrival_id, backends=backends):
self.format(departure)
self.flush()
def do_roadmap(self, line):
"""
roadmap DEPARTURE ARRIVAL
Display the roadmap to travel from DEPARTURE to ARRIVAL.
Command-line parameters:
--departure-time TIME requested departure time
--arrival-time TIME requested arrival time
TIME might be in form "yyyy-mm-dd HH:MM" or "HH:MM".
Example:
> roadmap Puteaux Aulnay-sous-Bois --arrival-time 22:00
"""
departure, arrival = self.parse_command_args(line, 2, 2)
filters = RoadmapFilters()
try:
filters.departure_time = self.parse_datetime(self.options.departure_time)
filters.arrival_time = self.parse_datetime(self.options.arrival_time)
except ValueError, e:
print >>sys.stderr, 'Invalid datetime value: %s' % e
print >>sys.stderr, 'Please enter a datetime in form "yyyy-mm-dd HH:MM" or "HH:MM".'
return 1
for backend, route in self.do('iter_roadmap', departure, arrival, filters):
self.format(route)
self.flush()
def parse_datetime(self, text):
if text is None:
return None
try:
date = datetime.strptime(text, '%Y-%m-%d %H:%M')
except ValueError:
try:
date = datetime.strptime(text, '%H:%M')
except ValueError:
raise ValueError(text)
date = datetime.now().replace(hour=date.hour, minute=date.minute)
return date
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/videoob/ 0000775 0000000 0000000 00000000000 11666415431 0026242 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/videoob/__init__.py 0000664 0000000 0000000 00000001426 11666415431 0030356 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .videoob import Videoob
__all__ = ['Videoob']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/videoob/videoob.py 0000664 0000000 0000000 00000017147 11666415431 0030255 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz, Romain Bignon, John Obbele, Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
import subprocess
import sys
import os
from weboob.capabilities.video import ICapVideo
from weboob.capabilities.base import NotLoaded
from weboob.tools.application.repl import ReplApplication
from weboob.tools.application.media_player import InvalidMediaPlayer, MediaPlayer, MediaPlayerNotFound
from weboob.tools.application.formatters.iformatter import IFormatter
__all__ = ['Videoob']
class VideoListFormatter(IFormatter):
MANDATORY_FIELDS = ('id', 'title', 'duration', 'date')
count = 0
def flush(self):
self.count = 0
pass
def format_dict(self, item):
self.count += 1
if self.interactive:
backend = item['id'].split('@', 1)[1]
result = u'%s* (%d) %s (%s)%s\n' % (self.BOLD, self.count, item['title'], backend, self.NC)
else:
result = u'%s* (%s) %s%s\n' % (self.BOLD, item['id'], item['title'], self.NC)
result += ' %s' % (item['duration'] if item['duration'] else item['date'])
if item['author'] is not NotLoaded:
result += ' - %s' % item['author']
if item['rating'] is not NotLoaded:
result += u' (%s/%s)' % (item['rating'], item['rating_max'])
return result
class Videoob(ReplApplication):
APPNAME = 'videoob'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Christophe Benz, Romain Bignon, John Obbele'
DESCRIPTION = 'Console application allowing to search for videos on various websites, ' \
'play and download them and get information.'
CAPS = ICapVideo
EXTRA_FORMATTERS = {'video_list': VideoListFormatter}
COMMANDS_FORMATTERS = {'search': 'video_list',
'ls': 'video_list'}
nsfw = True
def __init__(self, *args, **kwargs):
ReplApplication.__init__(self, *args, **kwargs)
self.player = MediaPlayer(self.logger)
def main(self, argv):
self.load_config()
return ReplApplication.main(self, argv)
def complete_download(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_object()
elif len(args) >= 3:
return self.path_completer(args[2])
def do_download(self, line):
"""
download ID [FILENAME]
Download a video
"""
_id, dest = self.parse_command_args(line, 2, 1)
video = self.get_object(_id, 'get_video', ['url'])
if not video:
print >>sys.stderr, 'Video not found: %s' % _id
return 3
if not video.url:
print >>sys.stderr, 'Error: the direct URL is not available.'
return 4
def check_exec(executable):
with open('/dev/null', 'w') as devnull:
process = subprocess.Popen(['which', executable], stdout=devnull)
if process.wait() != 0:
print >>sys.stderr, 'Please install "%s"' % executable
return False
return True
if dest is None:
ext = video.ext
if not ext:
ext = 'avi'
dest = '%s.%s' % (video.id, ext)
if video.url.find('rtmp') == 0:
if not check_exec('rtmpdump'):
return 1
args = ('rtmpdump', '-r', video.url, '-o', dest)
elif video.url.find('mms') == 0:
if not check_exec('mimms'):
return 1
args = ('mimms', video.url, dest)
else:
if not check_exec('wget'):
return 1
args = ('wget', video.url, '-O', dest)
os.spawnlp(os.P_WAIT, args[0], *args)
def complete_play(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_object()
def do_play(self, _id):
"""
play ID
Play a video with a found player.
"""
if not _id:
print >>sys.stderr, 'This command takes an argument: %s' % self.get_command_help('play', short=True)
return 2
video = self.get_object(_id, 'get_video', ['url'])
if not video:
print >>sys.stderr, 'Video not found: %s' % _id
return 3
if not video.url:
print >>sys.stderr, 'Error: the direct URL is not available.'
return 4
try:
player_name = self.config.get('media_player')
if not player_name:
self.logger.info(u'You can set the media_player key to the player you prefer in the videoob '
'configuration file.')
self.player.play(video, player_name=player_name)
except (InvalidMediaPlayer, MediaPlayerNotFound), e:
print '%s\nVideo URL: %s' % (e, video.url)
def complete_info(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_object()
def do_info(self, _id):
"""
info ID
Get information about a video.
"""
if not _id:
print >>sys.stderr, 'This command takes an argument: %s' % self.get_command_help('info', short=True)
return 2
video = self.get_object(_id, 'get_video')
if not video:
print >>sys.stderr, 'Video not found: %s' % _id
return 3
self.format(video)
self.flush()
def complete_nsfw(self, text, line, begidx, endidx):
return ['on', 'off']
def do_nsfw(self, line):
"""
nsfw [on | off]
If argument is given, enable or disable the non-suitable for work behavior.
If no argument is given, print the current behavior.
"""
line = line.strip()
if line:
if line == 'on':
self.nsfw = True
elif line == 'off':
self.nsfw = False
else:
print 'Invalid argument "%s".' % line
return 2
else:
print "on" if self.nsfw else "off"
def do_search(self, pattern=None):
"""
search [PATTERN]
Search for videos matching a PATTERN.
If PATTERN is not given, this command will search for the latest videos.
"""
if len(self.enabled_backends) == 0:
if self.interactive:
print >>sys.stderr, 'No backend loaded. Please use the "backends" command.'
else:
print >>sys.stderr, 'No backend loaded.'
return 1
self.set_formatter_header(u'Search pattern: %s' % pattern if pattern else u'Latest videos')
self.change_path('/search')
for backend, video in self.do('iter_search_results', pattern=pattern, nsfw=self.nsfw,
max_results=self.options.count):
self.add_object(video)
self.format(video)
self.flush()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/videoob_web/ 0000775 0000000 0000000 00000000000 11666415431 0027077 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/videoob_web/__init__.py 0000664 0000000 0000000 00000001440 11666415431 0031207 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .videoob_web import VideoobWeb
__all__ = ['VideoobWeb']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/videoob_web/public/ 0000775 0000000 0000000 00000000000 11666415431 0030355 5 ustar 00root root 0000000 0000000 style.css 0000664 0000000 0000000 00000000076 11666415431 0032153 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/videoob_web/public .video-item
{
margin-bottom: 5ex;
margin-left: 2em;
}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/videoob_web/templates/ 0000775 0000000 0000000 00000000000 11666415431 0031075 5 ustar 00root root 0000000 0000000 base.mako 0000664 0000000 0000000 00000000622 11666415431 0032601 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/videoob_web/templates ## -*- coding: utf-8 -*-
<%def name="title()" filter="trim">
Videoob Web
%def>
% if merge:
% for item in results:
${video_item(item)}
% endfor
% else:
% for backend, items in sorted(results.iteritems()):
${backend}
% for item in items:
${video_item(item)}
% endfor
% endfor
% endif
%def>
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/videoob_web/videoob_web.py 0000664 0000000 0000000 00000010520 11666415431 0031733 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import os
from mako.lookup import TemplateLookup
from mako.runtime import Context
from routes import Mapper
from StringIO import StringIO
from webob.dec import wsgify
from webob import exc
from wsgiref.simple_server import make_server
from weboob.capabilities.video import ICapVideo
from weboob.tools.application.base import BaseApplication
__all__ = ['VideoobWeb']
template_lookup = TemplateLookup(directories=[os.path.join(os.path.dirname(__file__), 'templates')],
output_encoding='utf-8', encoding_errors='replace')
class VideoobWeb(BaseApplication):
APPNAME = 'videoob-webserver'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Christophe Benz'
DESCRIPTION = 'WSGI web server application allowing to search for videos on various websites.'
CAPS = ICapVideo
CONFIG = dict(host='localhost', port=8080)
@wsgify
def make_app(self, req):
map = Mapper()
map.connect('index', '/', method='index')
results = map.routematch(environ=req.environ)
if results:
match, route = results
req.urlvars = ((), match)
kwargs = match.copy()
method = kwargs.pop('method')
return getattr(self, method)(req, **kwargs)
else:
public_path = os.path.join(os.path.dirname(__file__), 'public')
if not os.path.exists(public_path):
return exc.HTTPNotFound()
path = req.path
if path.startswith('/'):
path = path[1:]
public_file_path = os.path.join(public_path, path)
if os.path.exists(public_file_path):
if path.endswith('.css'):
req.response.content_type = 'text/css'
elif path.endswith('.js'):
req.response.content_type = 'text/javascript'
return open(public_file_path, 'r').read().strip()
else:
return exc.HTTPNotFound()
def main(self, argv):
self.load_config()
self.weboob.load_backends(ICapVideo)
print 'Web server created. Listening on http://%s:%s' % (
self.config.get('host'), int(self.config.get('port')))
srv = make_server(self.config.get('host'), int(self.config.get('port')), self.make_app)
srv.serve_forever()
def index(self, req):
c = {}
nsfw = req.params.get('nsfw')
nsfw = False if not nsfw or nsfw == '0' else True
q = req.params.get('q', u'')
merge = req.params.get('merge')
merge = False if not merge or merge == '0' else True
c['merge'] = merge
c['form_data'] = dict(q=q)
c['results'] = [] if merge else {}
if q:
for backend in self.weboob.iter_backends():
videos = [dict(title=video.title,
page_url=video.page_url,
url=video.url if video.url else '/download?id=%s' % video.id,
thumbnail_url=video.thumbnail_url,
) \
for video in backend.iter_search_results(pattern=q, nsfw=nsfw)]
if videos:
if merge:
c['results'].extend(videos)
else:
c['results'][backend.name] = videos
if merge:
c['results'] = sorted(c['results'], key=lambda video: video['title'].lower())
template = template_lookup.get_template('index.mako')
buf = StringIO()
ctx = Context(buf, **c)
template.render_context(ctx)
return buf.getvalue().strip()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/webcontentedit/ 0000775 0000000 0000000 00000000000 11666415431 0027631 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/webcontentedit/__init__.py 0000664 0000000 0000000 00000001451 11666415431 0031743 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .webcontentedit import WebContentEdit
__all__ = ['WebContentEdit']
webcontentedit.py 0000664 0000000 0000000 00000011331 11666415431 0033141 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/webcontentedit # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
# python2.5 compatibility
from __future__ import with_statement
import os
import sys
import tempfile
from weboob.core.bcall import CallErrors
from weboob.capabilities.content import ICapContent
from weboob.tools.application.repl import ReplApplication
__all__ = ['WebContentEdit']
class WebContentEdit(ReplApplication):
APPNAME = 'webcontentedit'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
DESCRIPTION = 'Console application allowing to display and edit contents on various websites.'
CAPS = ICapContent
def do_edit(self, line):
"""
edit ID [ID...]
Edit a content with $EDITOR, then push it on the website.
"""
contents = []
for id in line.split():
_id, backend_name = self.parse_id(id, unique_backend=True)
backend_names = (backend_name,) if backend_name is not None else self.enabled_backends
contents += [content for backend, content in self.do('get_content', _id, backends=backend_names) if content]
if len(contents) == 0:
print >>sys.stderr, 'No contents found'
return 3
paths = {}
for content in contents:
tmpdir = os.path.join(tempfile.gettempdir(), "weboob")
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
fd, path = tempfile.mkstemp(prefix='%s_' % content.id.replace(os.path.sep, '_'), dir=tmpdir)
with os.fdopen(fd, 'w') as f:
data = content.content
if isinstance(data, unicode):
data = data.encode('utf-8')
elif data is None:
content.content = u''
data = ''
f.write(data)
paths[path.encode('utf-8')] = content
params = ''
editor = os.environ.get('EDITOR', 'vim')
if editor == 'vim':
params = '-p'
os.system("%s %s %s" % (editor, params, ' '.join(['"%s"' % path.replace('"', '\\"') for path in paths.iterkeys()])))
for path, content in paths.iteritems():
with open(path, 'r') as f:
data = f.read()
try:
data = data.decode('utf-8')
except UnicodeError:
pass
if content.content != data:
content.content = data
else:
contents.remove(content)
if len(contents) == 0:
print >>sys.stderr, 'No changes. Abort.'
return 1
print 'Contents changed:\n%s' % ('\n'.join(' * %s' % content.id for content in contents))
message = self.ask('Enter a commit message', default='')
minor = self.ask('Is this a minor edit?', default=False)
if not self.ask('Do you want to push?', default=True):
return
errors = CallErrors([])
for content in contents:
path = [path for path, c in paths.iteritems() if c == content][0]
sys.stdout.write('Pushing %s...' % content.id.encode('utf-8'))
sys.stdout.flush()
try:
self.do('push_content', content, message, minor=minor, backends=[content.backend]).wait()
except CallErrors, e:
errors.errors += e.errors
sys.stdout.write(' error (content saved in %s)\n' % path)
else:
sys.stdout.write(' done\n')
os.unlink(path)
if len(errors.errors) > 0:
raise errors
def do_log(self, line):
"""
log ID
Display log of a page
"""
if not line:
print >>sys.stderr, 'Error: please give a page ID'
return 2
_id, backend_name = self.parse_id(line)
backend_names = (backend_name,) if backend_name is not None else self.enabled_backends
_id = _id.encode('utf-8')
for backend, revision in self.do('iter_revisions', _id, max_results=self.options.count, backends=backend_names):
self.format(revision)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/weboobcfg/ 0000775 0000000 0000000 00000000000 11666415431 0026550 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/weboobcfg/__init__.py 0000664 0000000 0000000 00000001432 11666415431 0030661 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .weboobcfg import WeboobCfg
__all__ = ['WeboobCfg']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/weboobcfg/weboobcfg.py 0000664 0000000 0000000 00000020657 11666415431 0031071 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import os
import sys
import re
from copy import copy
from weboob.capabilities.account import ICapAccount
from weboob.core.modules import ModuleLoadError
from weboob.tools.application.repl import ReplApplication
from weboob.tools.ordereddict import OrderedDict
__all__ = ['WeboobCfg']
class WeboobCfg(ReplApplication):
APPNAME = 'weboob-config'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Christophe Benz, Romain Bignon'
DESCRIPTION = "Weboob-Config is a console application to add/edit/remove backends, " \
"and to register new website accounts."
COMMANDS_FORMATTERS = {'backends': 'table',
'list': 'table',
}
DISABLE_REPL = True
weboob_commands = copy(ReplApplication.weboob_commands)
weboob_commands.remove('backends')
def load_default_backends(self):
pass
def do_add(self, line):
"""
add NAME [OPTIONS ...]
Add a configured backend.
"""
if not line:
print >>sys.stderr, 'You must specify a backend name. Hint: use the "backends" command.'
return 2
name, options = self.parse_command_args(line, 2, 1)
if options:
options = options.split(' ')
else:
options = ()
params = {}
# set backend params from command-line arguments
for option in options:
try:
key, value = option.split('=', 1)
except ValueError:
print >>sys.stderr, 'Parameters have to be formatted "key=value"'
return 2
params[key] = value
self.add_backend(name, params)
def do_register(self, line):
"""
register NAME
Register a new account on a backend.
"""
self.register_backend(line)
def do_confirm(self, backend_name):
"""
confirm BACKEND
For a backend which support CapAccount, parse a confirmation mail
after using the 'register' command to automatically confirm the
subscribe.
It takes mail from stdin. Use it with postfix for example.
"""
# Do not use the ReplApplication.load_backends() method because we
# don't want to prompt user to create backend.
self.weboob.load_backends(names=[backend_name])
try:
backend = self.weboob.get_backend(backend_name)
except KeyError:
print >>sys.stderr, 'Error: backend "%s" not found.' % backend_name
return 1
if not backend.has_caps(ICapAccount):
print >>sys.stderr, 'Error: backend "%s" does not support accounts management' % backend_name
return 1
mail = self.acquire_input()
if not backend.confirm_account(mail):
print >>sys.stderr, 'Error: Unable to confirm account creation'
return 1
return 0
def do_list(self, line):
"""
list [CAPS ..]
Show configured backends.
"""
caps = line.split()
for instance_name, name, params in sorted(self.weboob.backends_config.iter_backends()):
backend = self.weboob.modules_loader.get_or_load_module(name)
if caps and not self.caps_included(backend.iter_caps(), caps):
continue
row = OrderedDict([('Instance name', instance_name),
('Backend', name),
('Configuration', ', '.join(
'%s=%s' % (key, ('*****' if key in backend.config and backend.config[key].masked \
else value)) \
for key, value in params.iteritems())),
])
self.format(row)
self.flush()
def do_remove(self, instance_name):
"""
remove NAME
Remove a configured backend.
"""
if not self.weboob.backends_config.remove_backend(instance_name):
print >>sys.stderr, 'Backend instance "%s" does not exist' % instance_name
return 1
def _do_toggle(self, name, state):
try:
bname, items = self.weboob.backends_config.get_backend(name)
except KeyError:
print >>sys.stderr, 'Backend instance "%s" does not exist' % name
return 1
self.weboob.backends_config.edit_backend(name, bname, {'_enabled': state})
def do_enable(self, name):
"""
enable NAME
Enable a disabled backend
"""
self._do_toggle(name, 1)
def do_disable(self, name):
"""
disable NAME
Disable a backend
"""
self._do_toggle(name, 0)
def do_edit(self, line):
"""
edit NAME
Edit a backend
"""
try:
self.edit_backend(line)
except KeyError:
print >>sys.stderr, 'Error: backend "%s" not found' % line
return 1
def do_backends(self, line):
"""
backends [CAPS ...]
Show available backends.
"""
caps = line.split()
self.weboob.modules_loader.load_all()
for name, backend in sorted(self.weboob.modules_loader.loaded.iteritems()):
if caps and not self.caps_included(backend.iter_caps(), caps):
continue
row = OrderedDict([('Name', name),
('Capabilities', ', '.join(cap.__name__ for cap in backend.iter_caps())),
('Description', backend.description),
])
self.format(row)
self.flush()
def do_info(self, line):
"""
info NAME
Display information about a backend.
"""
if not line:
print >>sys.stderr, 'You must specify a backend name. Hint: use the "backends" command.'
return 2
try:
backend = self.weboob.modules_loader.get_or_load_module(line)
except ModuleLoadError:
backend = None
if not backend:
print >>sys.stderr, 'Backend "%s" does not exist.' % line
return 1
print '.------------------------------------------------------------------------------.'
print '| Backend %-68s |' % backend.name
print "+-----------------.------------------------------------------------------------'"
print '| Version | %s' % backend.version
print '| Maintainer | %s' % backend.maintainer
print '| License | %s' % backend.license
print '| Description | %s' % backend.description
print '| Capabilities | %s' % ', '.join([cap.__name__ for cap in backend.iter_caps()])
first = True
for key, field in backend.config.iteritems():
value = field.label
if not field.default is None:
value += ' (default: %s)' % field.default
if first:
print '| | '
print '| Configuration | %s: %s' % (key, value)
first = False
else:
print '| | %s: %s' % (key, value)
print "'-----------------'"
def do_applications(self, line):
"""
applications
Show applications.
"""
applications = set()
import weboob.applications
for path in weboob.applications.__path__:
regexp = re.compile('^%s/([\w\d_]+)$' % path)
for root, dirs, files in os.walk(path):
m = regexp.match(root)
if m and '__init__.py' in files:
applications.add(m.group(1))
print ' '.join(sorted(applications)).encode('utf-8')
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/weboobcli/ 0000775 0000000 0000000 00000000000 11666415431 0026560 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/weboobcli/__init__.py 0000664 0000000 0000000 00000001432 11666415431 0030671 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .weboobcli import WeboobCli
__all__ = ['WeboobCli']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/weboobcli/weboobcli.py 0000664 0000000 0000000 00000003254 11666415431 0031103 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import sys
from weboob.tools.application.repl import ReplApplication
__all__ = ['WeboobCli']
class WeboobCli(ReplApplication):
APPNAME = 'weboob-cli'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
SYNOPSIS = 'Usage: %prog [-dqv] [-b backends] [-cnfs] capability method [arguments..]\n'
SYNOPSIS += ' %prog [--help] [--version]'
DESCRIPTION = "Weboob-Cli is a console application to call a specific method on backends " \
"which implement the given capability."
DISABLE_REPL = True
def load_default_backends(self):
pass
def main(self, argv):
if len(argv) < 3:
print >>sys.stderr, "Syntax: %s capability method [args ..]" % argv[0]
return 2
cap_s = argv[1]
cmd = argv[2]
args = argv[3:]
self.load_backends(cap_s)
for backend, obj in self.do(cmd, *args):
self.format(obj)
return 0
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/weboobdebug/ 0000775 0000000 0000000 00000000000 11666415431 0027077 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/weboobdebug/__init__.py 0000664 0000000 0000000 00000001441 11666415431 0031210 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .weboobdebug import WeboobDebug
__all__ = ['WeboobDebug']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/weboobdebug/weboobdebug.py 0000664 0000000 0000000 00000003453 11666415431 0031742 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import sys
from weboob.tools.application.repl import ReplApplication
class WeboobDebug(ReplApplication):
APPNAME = 'weboobdebug'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Christophe Benz'
DESCRIPTION = "Weboob-Debug is a console application to debug backends."
def load_default_backends(self):
pass
def do_shell(self, backend_name):
"""
shell BACKEND
Debug a backend.
"""
try:
backend = self.weboob.load_backends(names=[backend_name])[backend_name]
except KeyError:
print >>sys.stderr, u'Unable to load backend "%s"' % backend_name
return 1
browser = backend.browser
from IPython.Shell import IPShellEmbed
shell = IPShellEmbed(argv=[])
locs = dict(backend=backend, browser=browser, application=self, weboob=self.weboob)
banner = 'Weboob debug shell\nBackend "%s" loaded.\nAvailable variables: %s' % (backend_name, locs)
shell.set_banner(shell.IP.BANNER + '\n\n' + banner)
shell(local_ns=locs, global_ns={})
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/weboorrents/ 0000775 0000000 0000000 00000000000 11666415431 0027164 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/weboorrents/__init__.py 0000664 0000000 0000000 00000001440 11666415431 0031274 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .weboorrents import Weboorrents
__all__ = ['Weboorrents']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/weboorrents/weboorrents.py 0000664 0000000 0000000 00000012664 11666415431 0032120 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
import sys
from weboob.capabilities.torrent import ICapTorrent
from weboob.tools.application.repl import ReplApplication
from weboob.tools.application.formatters.iformatter import IFormatter
__all__ = ['Weboorrents']
def sizeof_fmt(num):
for x in ['bytes','KB','MB','GB','TB']:
if num < 1024.0:
return "%-4.1f%s" % (num, x)
num /= 1024.0
class TorrentInfoFormatter(IFormatter):
MANDATORY_FIELDS = ('id', 'name', 'size', 'seeders', 'leechers', 'url', 'files', 'description')
def flush(self):
pass
def format_dict(self, item):
result = u'%s%s%s\n' % (self.BOLD, item['name'], self.NC)
result += 'ID: %s\n' % item['id']
result += 'Size: %s\n' % sizeof_fmt(item['size'])
result += 'Seeders: %s\n' % item['seeders']
result += 'Leechers: %s\n' % item['leechers']
result += 'URL: %s\n' % item['url']
if item['files']:
result += '\n%sFiles%s\n' % (self.BOLD, self.NC)
for f in item['files']:
result += ' * %s\n' % f
result += '\n%sDescription%s\n' % (self.BOLD, self.NC)
result += item['description']
return result
class TorrentListFormatter(IFormatter):
MANDATORY_FIELDS = ('id', 'name', 'size', 'seeders', 'leechers')
count = 0
def flush(self):
self.count = 0
pass
def format_dict(self, item):
self.count += 1
if self.interactive:
backend = item['id'].split('@', 1)[1]
result = u'%s* (%d) %s (%s)%s\n' % (self.BOLD, self.count, item['name'], backend, self.NC)
else:
result = u'%s* (%s) %s%s\n' % (self.BOLD, item['id'], item['name'], self.NC)
size = sizeof_fmt(item['size'])
result += ' %10s (Seed: %2d / Leech: %2d)' % (size, item['seeders'], item['leechers'])
return result
class Weboorrents(ReplApplication):
APPNAME = 'weboorrents'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
DESCRIPTION = 'Console application allowing to search for torrents on various trackers ' \
'and download .torrent files.'
CAPS = ICapTorrent
EXTRA_FORMATTERS = {'torrent_list': TorrentListFormatter,
'torrent_info': TorrentInfoFormatter,
}
COMMANDS_FORMATTERS = {'search': 'torrent_list',
'info': 'torrent_info',
}
def complete_info(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_object()
def do_info(self, id):
"""
info ID
Get information about a torrent.
"""
_id, backend_name = self.parse_id(id)
found = 0
for backend, torrent in self.do('get_torrent', _id, backends=backend_name):
if torrent:
self.format(torrent)
found = 1
if not found:
print >>sys.stderr, 'Torrent "%s" not found' % id
return 3
else:
self.flush()
def complete_getfile(self, text, line, *ignored):
args = line.split(' ', 2)
if len(args) == 2:
return self._complete_object()
elif len(args) >= 3:
return self.path_completer(args[2])
def do_getfile(self, line):
"""
getfile ID [FILENAME]
Get the .torrent file.
FILENAME is where to write the file. If FILENAME is '-',
the file is written to stdout.
"""
id, dest = self.parse_command_args(line, 2, 1)
_id, backend_name = self.parse_id(id)
if dest is None:
dest = '%s.torrent' % _id
for backend, buf in self.do('get_torrent_file', _id, backends=backend_name):
if buf:
if dest == '-':
print buf
else:
try:
with open(dest, 'w') as f:
f.write(buf)
except IOError, e:
print >>sys.stderr, 'Unable to write .torrent in "%s": %s' % (dest, e)
return 1
return
print >>sys.stderr, 'Torrent "%s" not found' % id
return 3
def do_search(self, pattern):
"""
search [PATTERN]
Search torrents.
"""
self.change_path('/search')
if not pattern:
pattern = None
self.set_formatter_header(u'Search pattern: %s' % pattern if pattern else u'Latest torrents')
for backend, torrent in self.do('iter_torrents', pattern=pattern):
self.add_object(torrent)
self.format(torrent)
self.flush()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/wetboobs/ 0000775 0000000 0000000 00000000000 11666415431 0026437 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/wetboobs/__init__.py 0000664 0000000 0000000 00000001427 11666415431 0030554 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .wetboobs import WetBoobs
__all__ = ['WetBoobs']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/applications/wetboobs/wetboobs.py 0000664 0000000 0000000 00000011601 11666415431 0030634 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import datetime
from weboob.capabilities.weather import ICapWeather
from weboob.tools.application.repl import ReplApplication
from weboob.tools.application.formatters.iformatter import IFormatter
__all__ = ['WetBoobs']
class ForecastsFormatter(IFormatter):
MANDATORY_FIELDS = ('id', 'date', 'low', 'high', 'unit')
def flush(self):
pass
def format_dict(self, item):
result = u'%s* %-15s%s (%s°%s - %s°%s)' % (self.BOLD, '%s:' % item['date'], self.NC, item['low'], item['unit'], item['high'], item['unit'])
if 'text' in item and item['text']:
result += ' %s' % item['text']
return result
class CurrentFormatter(IFormatter):
MANDATORY_FIELDS = ('id', 'date', 'temp')
def flush(self):
pass
def format_dict(self, item):
if isinstance(item['date'], datetime):
date = item['date'].strftime('%y-%m-%d %H:%M:%S')
else:
date = item['date']
result = u'%s%s%s: %s' % (self.BOLD, date, self.NC, item['temp'])
if 'unit' in item and item['unit']:
result += u'°%s' % item['unit']
if 'text' in item and item['text']:
result += u' - %s' % item['text']
return result
class CitiesFormatter(IFormatter):
MANDATORY_FIELDS = ('id', 'name')
count = 0
def flush(self):
self.count = 0
def format_dict(self, item):
self.count += 1
if self.interactive:
backend = item['id'].split('@', 1)[1]
result = u'%s* (%d) %s (%s)%s' % (self.BOLD, self.count, item['name'], backend, self.NC)
else:
result = u'%s* (%s) %s%s' % (self.BOLD, item['id'], item['name'], self.NC)
return result
class WetBoobs(ReplApplication):
APPNAME = 'wetboobs'
VERSION = '0.9.1'
COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
DESCRIPTION = 'Console application allowing to display weather and forecasts in your city.'
CAPS = ICapWeather
EXTRA_FORMATTERS = {'cities': CitiesFormatter,
'current': CurrentFormatter,
'forecasts': ForecastsFormatter,
}
COMMANDS_FORMATTERS = {'search': 'cities',
'current': 'current',
'forecasts': 'forecasts',
}
cities = []
def do_search(self, pattern):
"""
search PATTERN
Search cities.
"""
self.cities = []
for backend, city in self.do('iter_city_search', pattern):
self.format(city)
self.cities.append(city)
self.flush()
def parse_id(self, id):
if self.interactive:
try:
city = self.cities[int(id) - 1]
except (IndexError,ValueError):
pass
else:
id = '%s@%s' % (city.id, city.backend)
return ReplApplication.parse_id(self, id)
def _complete_id(self):
return ['%s@%s' % (city.id, city.backend) for city in self.cities]
def complete_current(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_id()
def do_current(self, line):
"""
current CITY_ID
Get current weather for specified city. Use the 'search' command to find
its ID.
"""
city, = self.parse_command_args(line, 1, 1)
_id, backend_name = self.parse_id(city)
for backend, current in self.do('get_current', _id, backends=backend_name):
if current:
self.format(current)
self.flush()
def complete_forecasts(self, text, line, *ignored):
args = line.split(' ')
if len(args) == 2:
return self._complete_id()
def do_forecasts(self, line):
"""
forecasts CITY_ID
Get forecasts for specified city. Use the 'search' command to find
its ID.
"""
city, = self.parse_command_args(line, 1, 1)
_id, backend_name = self.parse_id(city)
for backend, forecast in self.do('iter_forecast', _id, backends=backend_name):
self.format(forecast)
self.flush()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ 0000775 0000000 0000000 00000000000 11666415431 0023677 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/__init__.py 0000664 0000000 0000000 00000000001 11666415431 0025777 0 ustar 00root root 0000000 0000000
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/arte/ 0000775 0000000 0000000 00000000000 11666415431 0024632 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/arte/__init__.py 0000664 0000000 0000000 00000001433 11666415431 0026744 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import ArteBackend
__all__ = ['ArteBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/arte/backend.py 0000664 0000000 0000000 00000004663 11666415431 0026604 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.video import ICapVideo
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import Value
from .browser import ArteBrowser
from .video import ArteVideo
__all__ = ['ArteBackend']
class ArteBackend(BaseBackend, ICapVideo):
NAME = 'arte'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
DESCRIPTION = 'Arte french TV'
LICENSE = 'AGPLv3+'
CONFIG = BackendConfig(Value('lang', label='Lang of videos',
choices={'fr': 'French', 'de': 'Deutsch', 'en': 'English'}, default='fr'),
Value('quality', label='Quality of videos', choices=['hd', 'sd'], default='hd'))
BROWSER = ArteBrowser
def create_default_browser(self):
return self.create_browser(lang=self.config['lang'].get(), quality=self.config['quality'].get())
def get_video(self, _id):
with self.browser:
return self.browser.get_video(_id)
def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
with self.browser:
return self.browser.iter_search_results(pattern)
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
with self.browser:
video = self.browser.get_video(ArteVideo.id2url(video.id), video)
if 'thumbnail' in fields and video.thumbnail:
with self.browser:
video.thumbnail.data = self.browser.readurl(video.thumbnail.url)
return video
OBJECTS = {ArteVideo: fill_video}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/arte/browser.py 0000664 0000000 0000000 00000004014 11666415431 0026666 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser
from weboob.tools.browser.decorators import id2url
from .pages import IndexPage, VideoPage
from .video import ArteVideo
__all__ = ['ArteBrowser']
class ArteBrowser(BaseBrowser):
DOMAIN = u'videos.arte.tv'
ENCODING = None
PAGES = {r'http://videos.arte.tv/\w+/videos/arte7.*': IndexPage,
r'http://videos.arte.tv/\w+/do_search/videos/.*': IndexPage,
r'http://videos.arte.tv/\w+/videos/(?P.+)\.html': VideoPage
}
SEARCH_LANG = {'fr': 'recherche', 'de':'suche', 'en': 'search'}
def __init__(self, lang, quality, *args, **kwargs):
BaseBrowser.__init__(self, *args, **kwargs)
self.lang = lang
self.quality = quality
@id2url(ArteVideo.id2url)
def get_video(self, url, video=None):
self.location(url)
return self.page.get_video(video, self.lang, self.quality)
def home(self):
self.location('http://videos.arte.tv/fr/videos/arte7')
def iter_search_results(self, pattern):
if not pattern:
self.home()
else:
self.location(self.buildurl('/%s/do_search/videos/%s' % (self.lang, self.SEARCH_LANG[self.lang]), q=pattern.encode('utf-8')))
assert self.is_on_page(IndexPage)
return self.page.iter_videos()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/arte/pages.py 0000664 0000000 0000000 00000007467 11666415431 0026321 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import datetime
import re
import urllib
from weboob.tools.browser import BasePage, BrokenPageError
from .video import ArteVideo
__all__ = ['IndexPage', 'VideoPage']
class IndexPage(BasePage):
def iter_videos(self):
videos = self.document.getroot().cssselect("div[class=video]")
for div in videos:
title = div.find('h2').find('a').text
m = re.match(r'/fr/videos/(.*)\.html', div.find('h2').find('a').attrib['href'])
_id = ''
if m:
_id = m.group(1)
rating = rating_max = 0
rates = self.parser.select(div, 'div[class=rateContainer]', 1)
for r in rates.findall('div'):
if 'star-rating-on' in r.attrib['class']:
rating += 1
rating_max += 1
video = ArteVideo(_id)
video.title = title
video.rating = rating
video.rating_max = rating_max
thumb = self.parser.select(div, 'img[class=thumbnail]', 1)
video.thumbnail_url = 'http://videos.arte.tv' + thumb.attrib['src']
try:
parts = self.parser.select(div, 'div.duration_thumbnail', 1).text.split(':')
if len(parts) == 2:
hours = 0
minutes, seconds = parts
elif len(parts) == 3:
hours, minutes, seconds = parts
else:
raise BrokenPageError('Unable to parse duration %r' % parts)
except BrokenPageError:
pass
else:
video.duration = datetime.timedelta(hours=int(hours), minutes=int(minutes), seconds=int(seconds))
yield video
class VideoPage(BasePage):
def get_video(self, video=None, lang='fr', quality='hd'):
if not video:
video = ArteVideo(self.group_dict['id'])
video.title = self.get_title()
video.url = self.get_url(lang, quality)
return video
def get_title(self):
return self.document.getroot().cssselect('h2')[0].text
def get_url(self, lang, quality):
obj = self.parser.select(self.document.getroot(), 'object', 1)
movie_url = self.parser.select(obj, 'param[name=movie]', 1)
xml_url = urllib.unquote(movie_url.attrib['value'].split('videorefFileUrl=')[-1])
doc = self.browser.get_document(self.browser.openurl(xml_url))
videos_list = self.parser.select(doc.getroot(), 'video')
videos = {}
for v in videos_list:
videos[v.attrib['lang']] = v.attrib['ref']
if lang in videos:
xml_url = videos[lang]
else:
xml_url = videos.popitem()[1]
doc = self.browser.get_document(self.browser.openurl(xml_url))
obj = self.parser.select(doc.getroot(), 'urls', 1)
videos_list = self.parser.select(obj, 'url')
urls = {}
for v in videos_list:
urls[v.attrib['quality']] = v.text
if quality in urls:
video_url = urls[quality]
else:
video_url = urls.popitem()[1]
return video_url
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/arte/test.py 0000664 0000000 0000000 00000002143 11666415431 0026163 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class ArteTest(BackendTest):
BACKEND = 'arte'
def test_arte(self):
l = list(self.backend.iter_search_results('arte'))
if len(l) > 0:
v = l[0]
self.backend.fillobj(v, ('url',))
self.assertTrue(v.url and v.url.startswith('rtmp://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/arte/video.py 0000664 0000000 0000000 00000001664 11666415431 0026321 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.video import BaseVideo
__all__ = ['ArteVideo']
class ArteVideo(BaseVideo):
@classmethod
def id2url(cls, _id):
return 'http://videos.arte.tv/fr/videos/%s.html' % _id
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/ 0000775 0000000 0000000 00000000000 11666415431 0024461 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/API.txt 0000664 0000000 0000000 00000127205 11666415431 0025642 0 ustar 00root root 0000000 0000000 Adopte un Mec API
------------------
Constants:
APIKEY = fb0123456789abcd
URL = http://api.adopteunmec.com/api.php
ME Commands
===========
me.login
---------
Parameters:
- login
- pass
Errors:
- 1.1.1 : invalid login
Return value:
{u'errors': [],
u'result': {u'baskets': u'384',
u'events': {u'lastBasket': {u'alert': u'1',
u'birthday': u'1989-02-04',
u'city': u'Paris',
u'country': u'fr',
u'cover': u'1',
u'id': u'14471939',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2011-09-20 13:56:36',
u'list5': u'0',
u'login': u'kloo',
u'mod_level': u'0',
u'path': u'9/3/9/1/7/4/4/',
u'pseudo': u'Kloolloo',
u'region': u'11',
u'sex': 1,
u'shard': 9,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/14471939/Kloolloo',
u'zip': u'75001'},
u'lastChat': {u'alert': u'1',
u'birthday': u'1987-10-10',
u'city': u'Paris 18e Arrondissement',
u'country': u'fr',
u'cover': u'0',
u'id': u'13280274',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2010-11-20 03:37:00',
u'list5': u'0',
u'login': u'#c1a63bda81cc03ccdf080ca6e003919e',
u'mod_level': u'0',
u'path': u'4/7/2/0/8/2/3/',
u'pseudo': u'Katie Lee',
u'region': u'11',
u'sex': 1,
u'shard': 4,
u'style': u'0',
u'table': u'adopteun.girls_coma',
u'url': u'/api.php?member/view/13280274/KatieLee',
u'zip': u'75018'},
u'lastFlash': {u'alert': u'1',
u'birthday': u'1983-12-20',
u'city': u'Longjumeau',
u'country': u'fr',
u'cover': u'1',
u'id': u'13883475',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-20 18:52:13',
u'list5': u'0',
u'login': u'@',
u'mod_level': u'0',
u'path': u'5/7/4/3/8/8/3/',
u'pseudo': u'Pas Tjs Sage',
u'region': u'11',
u'sex': 1,
u'shard': 5,
u'style': u'2',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/13883475/PasTjsSage',
u'zip': u'91160'},
u'lastMail': {u'alert': u'1',
u'birthday': u'1989-04-28',
u'city': u'Paris 8e Arrondissement',
u'country': u'fr',
u'cover': u'5',
u'id': u'13268738',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-20 19:24:14',
u'list5': u'0',
u'login': u'@13268738',
u'mod_level': u'0',
u'path': u'8/3/7/8/6/2/3/',
u'pseudo': u'Th\xe9na',
u'region': u'11',
u'sex': 1,
u'shard': 8,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/13268738/Thna',
u'zip': u'75008'},
u'lastVisit': {u'alert': u'1',
u'birthday': u'1988-02-27',
u'city': u'Paris',
u'country': u'fr',
u'cover': u'1',
u'id': u'14477637',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-20 18:31:00',
u'list5': u'0',
u'login': u'@',
u'mod_level': u'0',
u'path': u'7/3/6/7/7/4/4/',
u'pseudo': u'Lolly',
u'region': u'11',
u'sex': 1,
u'shard': 7,
u'style': u'4',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/14477637/Lolly',
u'zip': u'75005'}},
u'flashs': 10,
u'mails': u'731',
u'me': {u'about1': u"Je n'ai pas de temps à perdre, je n'ai ni MSN ni Facebook, je considère qu'on apprécie davantage la discussion face à face autour d'un verre que dans les yeux de son écran.\r \r Bon et ne venez que si vous avez quelque chose à me dire, je n'envoie jamais de charmes.",
u'about2': u'',
u'admin': u'0',
u'alert': u'0',
u'alert_add': u'6',
u'birthday': u'1986-08-13',
u'books': u"Orwell (1984, La ferme des animaux) Barjavel (La nuit des temps, Le voyageur imprudent, Ravage, \x85) Boris Vian (J'irai cracher sur vos tombes, L'écume des jours\x85) Bukowski, Desproges, San Antonio Sartre, Le Canard",
u'cat': u'1',
u'checks1': u'0',
u'checks2': u'16686',
u'checks3': u'0',
u'checks4': u'0',
u'checks5': u'0',
u'checks6': u'2',
u'checks7': u'0',
u'cinema': u'Le Grand Détournement \x97 La Classe Américaine V pour Vendetta, Pulp Fiction, The Truman Show Eternal Sunshine of the Spotless Mind, Match Point Idiocracy, The Big Lebowski, La cité de la peur Sin City, Orange Mecanique, Buffet Froid, L',
u'city': u'Paris',
u'country': u'fr',
u'cover': u'5',
u'drink': u'2',
u'email': u'tesiruna@parano.me',
u'eyes': u'3',
u'f': u'',
u'first_cnx': u'2010-05-16 09:13:53',
u'first_ip': u'81.57.125.104',
u'food': u'1',
u'godfather': u'0',
u'hair_color': u'5',
u'hair_size': u'3',
u'hobbies': u'',
u'id': u'22450639',
u'img_count': u'5',
u'isBan': False,
u'isOnline': True,
u'job': u'Dieu',
u'last_chat': u'-0001-11-29 23:09:21',
u'last_cnx': u'2011-09-20 19:24:25',
u'last_ip': u'88.161.27.232',
u'lat': u'48.861961',
u'latR': u'0.852802098431',
u'list1': u'2',
u'list2': u'2',
u'list3': u'2',
u'list4': u'3',
u'list5': u'0',
u'list6': u'0',
u'lng': u'2.33594',
u'lngR': u'0.040769844129',
u'login': u'@22450639',
u'mod_level': u'1',
u'music': u"Pink Floyd, Scorpions, Emperor Metallica, Iron Maiden, Accept Slash's Snakepit, Queen, Deep Purple Led Zeppelin, Rolling Stones Brassens, Souchon, Brel, Vian",
u'origins': u'1',
u'pass': u'8f3fa83cec9a243ae53c1337d2b5e1cf',
u'path': u'9/3/6/0/5/4/2/',
u'phone': u'-',
u'pictures': [{u'file': u'5',
u'height': u'427',
u'id': u'7315391',
u'md5': u'859fcd2e425617c33c16d6a1bc510ad3',
u'member': u'22450639',
u'rank': u'1',
u'valid': u'1578737',
u'width': u'500'}],
u'pseudo': u'Nazification',
u'region': u'11',
u'sex': 0,
u'shape': u'1',
u'shard': 9,
u'size': u'175',
u'smoke': u'2',
u'style': u'0',
u'subregion': u'76',
u'table': u'adopteun.boys',
u'texts1': u'',
u'texts2': u'',
u'texts3': u'',
u'texts4': u'',
u'texts5': u'',
u'texts6': u'',
u'title': u'',
u'tvs': u'Je ne possède pas la TV
',
u'url': u'/api.php?member/view/22450639/Nazification',
u'validated': True,
u'visites': u'0',
u'w': u'',
u'warn': u'0',
u'weight': u'55',
u'zip': u'75000'},
u'news': {u'newBaskets': 3, u'newMails': 1, u'newVisits': 113},
u'popu': u'71540',
u'subMobile': False,
u'subWebsite': False,
u'token': u'ea7bac8837f6e5bb1e2a3267d6a08b32e388d593',
u'visites': u'958'}}
me.[default]
------------
Return value:
{u'errors': [],
u'result': {u'events': {u'lastBasket': {u'alert': u'1',
u'birthday': u'1989-02-04',
u'city': u'Paris',
u'country': u'fr',
u'cover': u'1',
u'id': u'14471939',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2011-09-20 13:56:36',
u'list5': u'0',
u'login': u'kloo',
u'mod_level': u'0',
u'path': u'9/3/9/1/7/4/4/',
u'pseudo': u'Kloolloo',
u'region': u'11',
u'sex': 1,
u'shard': 9,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/14471939/Kloolloo',
u'zip': u'75001'},
u'lastChat': {u'alert': u'1',
u'birthday': u'1987-10-10',
u'city': u'Paris 18e Arrondissement',
u'country': u'fr',
u'cover': u'0',
u'id': u'13280274',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2010-11-20 03:37:00',
u'list5': u'0',
u'login': u'#c1a63bda81cc03ccdf080ca6e003919e',
u'mod_level': u'0',
u'path': u'4/7/2/0/8/2/3/',
u'pseudo': u'Katie Lee',
u'region': u'11',
u'sex': 1,
u'shard': 4,
u'style': u'0',
u'table': u'adopteun.girls_coma',
u'url': u'/api.php?member/view/13280274/KatieLee',
u'zip': u'75018'},
u'lastFlash': {u'alert': u'1',
u'birthday': u'1983-12-20',
u'city': u'Longjumeau',
u'country': u'fr',
u'cover': u'1',
u'id': u'13883475',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-20 18:52:13',
u'list5': u'0',
u'login': u'@',
u'mod_level': u'0',
u'path': u'5/7/4/3/8/8/3/',
u'pseudo': u'Pas Tjs Sage',
u'region': u'11',
u'sex': 1,
u'shard': 5,
u'style': u'2',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/13883475/PasTjsSage',
u'zip': u'91160'},
u'lastMail': {u'alert': u'1',
u'birthday': u'1989-04-28',
u'city': u'Paris 8e Arrondissement',
u'country': u'fr',
u'cover': u'5',
u'id': u'13268738',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-20 19:47:28',
u'list5': u'0',
u'login': u'@13268738',
u'mod_level': u'0',
u'path': u'8/3/7/8/6/2/3/',
u'pseudo': u'Th\xe9na',
u'region': u'11',
u'sex': 1,
u'shard': 8,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/13268738/Thna',
u'zip': u'75008'},
u'lastVisit': {u'alert': u'1',
u'birthday': u'1988-02-27',
u'city': u'Paris',
u'country': u'fr',
u'cover': u'1',
u'id': u'14477637',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2011-09-20 19:09:00',
u'list5': u'0',
u'login': u'@',
u'mod_level': u'0',
u'path': u'7/3/6/7/7/4/4/',
u'pseudo': u'Lolly',
u'region': u'11',
u'sex': 1,
u'shard': 7,
u'style': u'4',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/14477637/Lolly',
u'zip': u'75005'}},
u'news': {u'newBaskets': 0, u'newMails': 1, u'newVisits': 113},
u'token': u'9a97a03774c9f440e676c78f48794a7221a67285'}}
me.baskets
----------
Return value:
{u'errors': [],
u'popu': 71840,
u'result': {u'basket': [{u'alert': u'1',
u'birthday': u'1989-02-04',
u'city': u'Paris',
u'country': u'fr',
u'cover': u'1',
u'date': u'2011-09-19 01:52:29',
u'id': u'14471939',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2011-09-20 13:56:36',
u'list5': u'0',
u'login': u'kloo',
u'mod_level': u'0',
u'path': u'9/3/9/1/7/4/4/',
u'pseudo': u'Kloolloo',
u'region': u'11',
u'sex': 1,
u'shard': 9,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/14471939/Kloolloo',
u'zip': u'75001'}],
u'inBasket': 57,
u'token': u'ea7bac8837f6e5bb1e2a3267d6a08b32e388d593'}}
MESSAGE Commands
================
message.[default]
-----------------
Arguments:
- P=,
Return value:
{u'errors': [],
u'result': {u'count': 1,
u'threads': [{u'cat': u'0',
u'date': u'2011-09-20 19:22:11',
u'id': u'11132125',
u'id_from': u'13268738',
u'id_to': u'22450639',
u'member': {u'alert': u'1',
u'birthday': u'1989-04-28',
u'city': u'Paris 8e Arrondissement',
u'country': u'fr',
u'cover': u'5',
u'id': u'13268738',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-20 19:54:15',
u'list5': u'0',
u'login': u'@13268738',
u'mod_level': u'0',
u'path': u'8/3/7/8/6/2/3/',
u'pseudo': u'Th\xe9na',
u'region': u'11',
u'sex': 1,
u'shard': 8,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/13268738/Thna',
u'zip': u'75008'},
u'message': u'0',
u'status': u'0',
u'title': u"J'aime bien les \xe9l\xe9phants. Toi ?"}],
u'token': u'0f70c31bc6d05d45bee64e6f2eab9b537640f2f8'}}
message.thread
--------------
Parameters:
- memberId
- count
Return value:
{u'result': {u'popu': u'15910',
u'thread': {u'isNew': False,
u'member': {u'alert': u'3',
u'birthday': u'1987-04-22',
u'city': u'Maisons-Alfort',
u'country': u'fr',
u'cover': u'6',
u'id': u'14022243',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2011-09-20 18:18:00',
u'list5': u'0',
u'login': u'@',
u'mod_level': u'0',
u'path': u'3/4/2/2/2/0/4/',
u'pseudo': u'Sophkipeut',
u'region': u'11',
u'sex': 1,
u'shard': 3,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/14022243/Sophkipeut',
u'zip': u'94700'},
u'messages': [{u'date': u'2011-09-19 17:52:03',
u'id': u'46583458',
u'id_from': u'14022243',
u'id_to': u'23185402',
u'message': u"Lol, moi j'ai fait attention, mais bon ça n'empêche pas son bidou, ceci dit c'est planqué par ses loooooooooooong poils ^^",
u'src': u'',
u'title': u"Lol, moi j'ai fait attention, mais bon \xe7a n'emp\xea..."},
{u'date': u'2011-09-19 17:49:39',
u'id': u'47467748',
u'id_from': u'23185402',
u'id_to': u'14022243',
u'message': u"Ah oui, justement le vétérinaire m'avait dit après la castration de Futex qu'il fallait faire attention à son poids, du coup ça m'a tellement vexé que j'ai fais attention au point qu'il est sans doute même trop maigre.",
u'src': u'',
u'title': u"Ah oui, justement le v\xe9t\xe9rinaire m'avait dit apr\xe8..."}],
u'remoteStatus': u'2',
u'status': u'1',
u'warning': 0},
u'token': u'dbeccf96256d4f11991626707881fdba28f54d73'}}
message.new
-----------
Parameters:
- memberId
- message
Return value:
{u'errors': [],
u'result': {u'thread': {u'isNew': False,
u'member': {u'alert': u'1',
u'birthday': u'1986-12-08',
u'city': u'Rosny-sous-Bois',
u'country': u'fr',
u'cover': u'6',
u'id': u'11099536',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2011-09-20 16:26:32',
u'list5': u'0',
u'login': u'@11099536',
u'mod_level': u'0',
u'path': u'6/3/5/9/9/0/1/',
u'pseudo': u'Debo',
u'region': u'11',
u'sex': 1,
u'shard': 6,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/11099536/Debo',
u'zip': u'93110'},
u'messages': [{u'date': u'2011-09-20 20:09:07',
u'id': u'46588573',
u'id_from': u'23185402',
u'id_to': u'11099536',
u'message': u'Coucou',
u'src': u'iphone',
u'title': u'Coucou'},
{u'date': u'2011-09-20 16:27:34',
u'id': u'46638469',
u'id_from': u'11099536',
u'id_to': u'23185402',
u'message': u"Coucou pour tout t'avouer je ne m'y etais pas connecté depuis septembre ! un peu moins de boulot alors j'y traine !!\r\n\r\nEt toi ça va? tu devais pas partir au canada? tu es deja revenu.?",
u'src': u'',
u'title': u"Coucou pour tout t'avouer je ne m'y etais pas co..."}],
u'remoteStatus': u'0',
u'status': u'2',
u'warning': 0},
u'token': u'2491f8ccb6741ceee2a461dc523939796904a0fa'}}
message.delete
--------------
Parameters:
- id_user
Return Value:
Unknown
MEMBER Commands
===============
member.view
-----------
Parameters:
- id
Return value:
{u'errors': [],
u'result': {u'member': {u'about1': u"comment te dire.. j'ai autant envie te laisser boire dans ma bouteille que mettre ma langue dans ta bouche !",
u'about2': u"LES BISOUS C'EST BIEN LES BIJOUX C'EST MIEUX!\r \r Et l'humour encore plus, mouhaha\r \r PS: Si vous êtes le sosie de Pharell Williams, adoptez moi ;)",
u'admin': u'0',
u'alert': u'1',
u'alert_add': u'0',
u'birthday': u'1991-05-31',
u'books': u"L'Analphabète - Rendell Etat limite - Assouline Marie-Antoinette - Zweig
",
u'cat': u'2',
u'checks1': u'2',
u'checks2': u'1588',
u'checks3': u'0',
u'checks4': u'0',
u'checks5': u'64',
u'checks6': u'192',
u'checks7': u'0',
u'cinema': u"My Blueberry Night Shinning - L'exorciste - Ester - Gothika How High ! - Requiem for a dream Remember Me - trainspotting He gots game - Buffet froid",
u'city': u'Paris',
u'country': u'fr',
u'cover': u'1',
u'drink': u'2',
u'eyes': u'5',
u'f': u'',
u'first_cnx': u'2011-09-17 00:18:59',
u'first_ip': u'82.120.134.233',
u'food': u'3',
u'godfather': u'0',
u'hair_color': u'4',
u'hair_size': u'3',
u'hobbies': u'Le théâtre définitivement ! et le sport !',
u'id': u'14465370',
u'img_count': u'6',
u'isBan': False,
u'isOnline': False,
u'job': u'etudiante',
u'last_chat': u'0000-00-00 00:00:00',
u'last_cnx': u'2011-09-24 00:09:29',
u'last_ip': u'92.151.177.164',
u'lat': u'48.8814',
u'latR': u'0.853141372984',
u'list1': u'0',
u'list2': u'42',
u'list3': u'0',
u'list4': u'0',
u'list5': u'0',
u'list6': u'0',
u'lng': u'2.3365',
u'lngR': u'0.0407796179728',
u'login': u'@',
u'mailable': True,
u'mod_level': u'0',
u'music': u'Daft Punk - Bloody Beetrots - Kid Cudi - Jamiroquai Red Hot - Ray Charles - BEP - Gorillaz - Birdy nam nam Citizen Cope - Angus & Julia Stone - Portishead 50cent - Sia - Ben Harper - Busta Rhymes Musiques de gansta ! ET Debussy',
u'origins': u'1',
u'path': u'0/7/3/5/6/4/4/',
u'phone': u'-',
u'popu': {u'bonus': u'6',
u'contacts': u'1',
u'flashs': u'246',
u'id': u'14465370',
u'invits': u'0',
u'mails': u'39',
u'popu': u'11345',
u'visites': u'645'},
u'pseudo': u'Ruslana',
u'region': u'11',
u'sex': 1,
u'shape': u'1',
u'shard': 0,
u'size': u'170',
u'smoke': u'2',
u'style': u'4',
u'subregion': u'76',
u'table': u'adopteun.girls',
u'texts1': u'',
u'texts2': u'',
u'texts3': u'',
u'texts4': u'',
u'texts5': u'',
u'texts6': u'MON SOURIRE HAHAHA',
u'title': u'',
u'tvs': u'Dexter OC True blood Envoyé spécial ;) - Arte ! ',
u'url': u'/api.php?member/view/14465370/Ruslana',
u'visites': u'0',
u'w': u'',
u'warn': u'0',
u'weight': u'50',
u'zip': u'75008'},
u'token': u'e0247704012e01bc32756b357b010e5206ac9c76'}}
member.pictures
---------------
Parameters:
- id
Return value:
{u'errors': [],
u'result': {u'pictures': [{u'id': u'12363004',
u'rating': 4.4473684210500002,
u'url': u'http://s0.adopteunmec.com/0/6/0/1/6/image7.jpg'}],
u'token': u'e131f1b194f2a19337882398b10b79457a638252'}}
member.addBasket
----------------
Parameters:
- id
Errors:
- 5.1.1 : member does not exist
- 5.1.5 : already sent charm to this one
- 5.1.6 : no enough charms available
Return value:
{u'errors': u'0',
u'flashs': 4,
u'result': {u'token': u'55039d0557393bb7c5e4381792143d003f0e60c0'}}
SEARH Commands
==============
search.[default]
----------------
Return value:
{u'errors': [],
u'result': {u'qsearch': {u'ageMax': u'25',
u'ageMin': u'18',
u'dist': u'0',
u'new': u'0',
u'query': u'{"sex":1,"ageMin":"18","ageMax":"25","region":"fr","new":"0","dist":"0"}',
u'region': u'fr',
u'sex': 1},
u'search': {u'ageMax': u'27',
u'ageMin': u'20',
u'checks1': u'0',
u'checks2': u'0',
u'country': u'fr',
u'dist': u'50',
u'drink': u'0',
u'eyes': u'0',
u'food': u'0',
u'hair_color': u'0',
u'hair_size': u'0',
u'origins': u'0',
u'pseudo': u'',
u'query': u'{"ageMin":"20","ageMax":"27","country":"fr","region":"11","subregion":"0","dist":"50","pseudo":"","sex":"1","sizeMin":"0","sizeMax":"0","weightMin":"0","weightMax":"75","shape":"0","hair_size":"0","hair_color":"0","eyes":"0","origins":"0","style":"0","checks1":"0","checks2":"0","smoke":"0","drink":"0","food":"0","search":"true"}',
u'region': u'11',
u'search': u'true',
u'sex': u'1',
u'shape': u'0',
u'sizeMax': u'0',
u'sizeMin': u'0',
u'smoke': u'0',
u'style': u'0',
u'subregion': u'0',
u'weightMax': u'75',
u'weightMin': u'0'},
u'token': u'3196a3365d927f2ee8738ec8dfc4a5abd75e3ee3'}}
search.quick
------------
Parameters:
- sex (int[0,1])
- ageMin (int)
- ageMax (int)
- region (str)
- new (int)
- dist (int)
Return Value:
{u'errors': [],
u'result': {u'regions': {u'be': None,
u'be_24': u' - wallonie',
u'be_25': u' - bruxelles capitale',
u'be_26': u' - flandre',
u'ca': None,
u'ca_34': u' - canada',
u'ca_35': u' - quebec',
u'ch': None,
u'ch_27': u' - r\xe9gion l\xe9manique',
u'ch_28': u' - espace Mittelland',
u'ch_29': u' - suisse du nord-ouest',
u'ch_30': u' - zurich',
u'ch_31': u' - suisse orientale',
u'ch_32': u' - suisse centrale',
u'ch_33': u' - tessin',
u'fr': None,
u'fr_1': u' - alsace',
u'fr_10': u' - haute-normandie',
u'fr_11': u' - ile-de-france',
u'fr_12': u' - languedoc-roussillon',
u'fr_13': u' - limousin',
u'fr_14': u' - lorraine',
u'fr_15': u' - midi-pyr\xe9n\xe9es',
u'fr_16': u' - nord-pas-de-calais',
u'fr_17': u' - paca',
u'fr_18': u' - pays de la loire',
u'fr_19': u' - picardie',
u'fr_2': u' - aquitaine',
u'fr_20': u' - poitou-charentes',
u'fr_21': u' - rh\xf4ne-alpes',
u'fr_22': u' - corse',
u'fr_23': u' - dom+tom',
u'fr_3': u' - auvergne',
u'fr_4': u' - basse-normandie',
u'fr_5': u' - bourgogne',
u'fr_6': u' - bretagne',
u'fr_7': u' - centre',
u'fr_8': u' - champagne-ardenne',
u'fr_9': u' - franche-comt\xe9'},
u'search': [{u'alert': u'2',
u'birthday': u'1985-11-21',
u'city': u'Boulogne-Billancourt',
u'country': u'fr',
u'cover': u'12',
u'id': u'14252744',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-24 15:19:48',
u'list5': u'0',
u'login': u'@',
u'mod_level': u'0',
u'path': u'4/4/7/2/5/2/4/',
u'pseudo': u'Birdy',
u'region': u'11',
u'sex': 1,
u'shard': 4,
u'style': u'5',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/14252744/Birdy',
u'zip': u'92100'}],
u'token': u'ef78ac6d812ab15f2f8efd578f4da4ef2e23aa71'}}
search.advanced
---------------
Parameters:
- ageMin (int)
- ageMax (int)
- country (str)
- region (int)
- subregion (int)
- dist (int)
- pseudo (str)
- sex (int[0,1])
- sizeMin (int)
- sizeMax (int)
- weightMin (int)
- weightMax (int)
- shape (int)
- hair_size (int)
- hair_color (int)
- eyes (int)
- origins (int)
- style (int)
- checks1 (int)
- checks2 (int)
- smoke (int)
- drink (int)
- food (int)
- search (bool)
Return Value:
{u'errors': [],
u'regions': {u'be': None,
u'be_24': u' - wallonie',
u'be_25': u' - bruxelles capitale',
u'be_26': u' - flandre',
u'ca': None,
u'ca_34': u' - canada',
u'ca_35': u' - quebec',
u'ch': None,
u'ch_27': u' - r\xe9gion l\xe9manique',
u'ch_28': u' - espace Mittelland',
u'ch_29': u' - suisse du nord-ouest',
u'ch_30': u' - zurich',
u'ch_31': u' - suisse orientale',
u'ch_32': u' - suisse centrale',
u'ch_33': u' - tessin',
u'fr': None,
u'fr_1': u' - alsace',
u'fr_10': u' - haute-normandie',
u'fr_11': u' - ile-de-france',
u'fr_12': u' - languedoc-roussillon',
u'fr_13': u' - limousin',
u'fr_14': u' - lorraine',
u'fr_15': u' - midi-pyr\xe9n\xe9es',
u'fr_16': u' - nord-pas-de-calais',
u'fr_17': u' - paca',
u'fr_18': u' - pays de la loire',
u'fr_19': u' - picardie',
u'fr_2': u' - aquitaine',
u'fr_20': u' - poitou-charentes',
u'fr_21': u' - rh\xf4ne-alpes',
u'fr_22': u' - corse',
u'fr_23': u' - dom+tom',
u'fr_3': u' - auvergne',
u'fr_4': u' - basse-normandie',
u'fr_5': u' - bourgogne',
u'fr_6': u' - bretagne',
u'fr_7': u' - centre',
u'fr_8': u' - champagne-ardenne',
u'fr_9': u' - franche-comt\xe9'},
u'result': {u'search': [{u'alert': u'1',
u'birthday': u'1988-04-07',
u'city': u'Dammartin',
u'country': u'fr',
u'cover': u'25',
u'id': u'13579115',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-24 15:17:22',
u'list5': u'0',
u'login': u'@',
u'mod_level': u'0',
u'path': u'5/1/1/9/7/5/3/',
u'pseudo': u"S\xe9 's\xe9",
u'region': u'11',
u'sex': 1,
u'shard': 5,
u'style': u'1',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/13579115/Ss',
u'zip': u'77230'}],
u'token': u'f92aede46118dcdba6d484146b4627777fbe7188'}}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/__init__.py 0000664 0000000 0000000 00000001510 11666415431 0026567 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .browser import AuMBrowser
from .backend import AuMBackend
__all__ = ['AuMBrowser', 'AuMBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/antispam.py 0000664 0000000 0000000 00000010170 11666415431 0026646 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
__all__ = ['AntiSpam']
class AntiSpam(object):
def check_thread(self, thread):
resume = thread['title']
# Check if there is an email address in the offer.
if re.match('^[\w\d\.\-_]+@[\w\d\.]+ vous offre la pos', resume):
return False
if thread['member']['pseudo'] == 'Ekaterina':
return False
return True
def check_profile(self, profile):
# The name of profile is in form #123456789
if profile['pseudo'] == '':
return False
if len(profile['about1'].strip()) > 30 and profile['about1'].strip() == profile['about2'].strip():
return False
if profile['about1'].startswith('salut! je te donne mon msn'):
return False
if profile['about2'].startswith('cam to cam'):
return False
if profile['about2'].startswith('je suis une femme tres tres belle et je recherche un homme qui aime le sexe'):
return False
if profile['about2'].endswith('mmmmmmmmmmmmmmmm'):
return False
for ipaddr in (profile['last_ip'], profile['first_ip']):
if ipaddr.startswith('41.202.'):
return False
if ipaddr.startswith('41.250.'):
return False
if ipaddr.startswith('41.141.'):
return False
if ipaddr.startswith('194.177.'):
return False
if re.match('105\.13\d.*', ipaddr):
return False
if ipaddr in ('62.157.186.18', '198.36.222.8', '212.234.67.61'):
return False
return True
def check_contact(self, contact):
if not self.check_profile(contact.aum_profile):
return False
first_ip = contact.profile['info']['IPaddr'].value.split(' ')[0]
last_ip = contact.profile['info']['IPaddr'].value.rstrip(')')
for ipaddr in (first_ip, last_ip):
if ipaddr.endswith('.afnet.net'):
return False
if ipaddr.endswith('.iam.net.ma'):
return False
if ipaddr.endswith('.amsterdam.ananoos.net'):
return False
if ipaddr.endswith('.tedata.net'):
return False
if ipaddr.endswith('kupo.fr'):
return False
if ipaddr.endswith('.static.virginmedia.com'):
return False
if ipaddr.endswith('frozenway.com'):
return False
if ipaddr.endswith('.rev.bgtn.net'):
return False
if ipaddr.endswith('real-vpn.com'):
return False
if ipaddr.endswith('.nl.ipodah.net'):
return False
if ipaddr.endswith('.wanamaroc.com'):
return False
if ipaddr.endswith('.ukservers.com'):
return False
if ipaddr.endswith('.startdedicated.com'):
return False
if ipaddr.endswith('.clients.your-server.de'):
return False
if ipaddr.endswith('.cba.embratel.net.br'):
return False
return True
def check_mail(self, mail):
# Spambot with a long first-message.
if mail['message'].find('Je veux que vous m\'ayez ecrit directement sur le mon e-mail') >= 0:
return False
if mail['message'].find('ilusa12010@live.fr') >= 0:
return False
return True
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/backend.py 0000664 0000000 0000000 00000050206 11666415431 0026425 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
import email
import time
import re
import datetime
from html2text import unescape
from dateutil import tz
from dateutil.parser import parse as _parse_dt
from weboob.capabilities.base import NotLoaded
from weboob.capabilities.chat import ICapChat
from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Message, Thread
from weboob.capabilities.dating import ICapDating, OptimizationNotFound
from weboob.capabilities.contact import ICapContact, ContactPhoto, Query, QueryError
from weboob.capabilities.account import ICapAccount, StatusField
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.browser import BrowserUnavailable
from weboob.tools.value import Value, ValuesDict, ValueBool, ValueBackendPassword
from weboob.tools.log import getLogger
from weboob.tools.misc import local2utc
from .contact import Contact
from .captcha import CaptchaError
from .antispam import AntiSpam
from .browser import AuMBrowser
from .optim.profiles_walker import ProfilesWalker
from .optim.visibility import Visibility
from .optim.queries_queue import QueriesQueue
__all__ = ['AuMBackend']
def parse_dt(s):
d = _parse_dt(s)
return local2utc(d)
class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapChat, ICapContact, ICapAccount):
NAME = 'aum'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = u"“Adopte un mec” french dating website"
CONFIG = BackendConfig(Value('username', label='Username'),
ValueBackendPassword('password', label='Password'),
ValueBool('antispam', label='Enable anti-spam', default=False),
ValueBool('baskets', label='Get baskets with new messages', default=True))
STORAGE = {'profiles_walker': {'viewed': []},
'queries_queue': {'queue': []},
'sluts': {},
}
BROWSER = AuMBrowser
MAGIC_ID_BASKET = 1
def __init__(self, *args, **kwargs):
BaseBackend.__init__(self, *args, **kwargs)
if self.config['antispam'].get():
self.antispam = AntiSpam()
else:
self.antispam = None
def create_default_browser(self):
return self.create_browser(self.config['username'].get(), self.config['password'].get())
def report_spam(self, id):
with self.browser:
self.browser.delete_thread(id)
# Do not report fakes to website, to let them to other guys :)
#self.browser.report_fake(id)
# ---- ICapDating methods ---------------------
def init_optimizations(self):
self.add_optimization('PROFILE_WALKER', ProfilesWalker(self.weboob.scheduler, self.storage, self.browser))
self.add_optimization('VISIBILITY', Visibility(self.weboob.scheduler, self.browser))
self.add_optimization('QUERIES_QUEUE', QueriesQueue(self.weboob.scheduler, self.storage, self.browser))
# ---- ICapMessages methods ---------------------
def fill_thread(self, thread, fields):
return self.get_thread(thread)
def iter_threads(self):
with self.browser:
threads = self.browser.get_threads_list()
for thread in threads:
if thread['member'].get('isBan', True):
with self.browser:
self.browser.delete_thread(thread['member']['id'])
continue
if self.antispam and not self.antispam.check_thread(thread):
self.logger.info('Skipped a spam-thread from %s' % thread['pseudo'])
self.report_spam(thread['member']['id'])
continue
t = Thread(int(thread['member']['id']))
t.flags = Thread.IS_DISCUSSION
t.title = 'Discussion with %s' % thread['member']['pseudo']
yield t
def get_thread(self, id, contacts=None, get_profiles=False):
"""
Get a thread and its messages.
The 'contacts' parameters is only used for internal calls.
"""
thread = None
if isinstance(id, Thread):
thread = id
id = thread.id
if not thread:
thread = Thread(int(id))
thread.flags = Thread.IS_DISCUSSION
full = False
else:
full = True
with self.browser:
mails = self.browser.get_thread_mails(id, 100)
my_name = self.browser.get_my_name()
child = None
msg = None
slut = self._get_slut(id)
if contacts is None:
contacts = {}
if not thread.title:
thread.title = u'Discussion with %s' % mails['member']['pseudo']
self.storage.set('sluts', thread.id, 'status', mails['status'])
self.storage.save()
for mail in mails['messages']:
flags = 0
if self.antispam and not self.antispam.check_mail(mail):
self.logger.info('Skipped a spam-mail from %s' % mails['member']['pseudo'])
self.report_spam(thread.id)
break
if parse_dt(mail['date']) > slut['lastmsg']:
flags |= Message.IS_UNREAD
if get_profiles:
if not mail['id_from'] in contacts:
with self.browser:
contacts[mail['id_from']] = self.get_contact(mail['id_from'])
if self.antispam and not self.antispam.check_contact(contacts[mail['id_from']]):
self.logger.info('Skipped a spam-mail-profile from %s' % mails['member']['pseudo'])
self.report_spam(thread.id)
break
if int(mail['id_from']) == self.browser.my_id:
if int(mails['remoteStatus']) == 0 and msg is None:
flags |= Message.IS_NOT_ACCUSED
else:
flags |= Message.IS_ACCUSED
msg = Message(thread=thread,
id=int(time.strftime('%Y%m%d%H%M%S', parse_dt(mail['date']).timetuple())),
title=thread.title,
sender=my_name if int(mail['id_from']) == self.browser.my_id else mails['member']['pseudo'],
receivers=[my_name if int(mail['id_from']) != self.browser.my_id else mails['member']['pseudo']],
date=parse_dt(mail['date']),
content=unescape(mail['message']).strip(),
signature=contacts[mail['id_from']].get_text() if mail['id_from'] in contacts else None,
children=[],
flags=flags)
if child:
msg.children.append(child)
child.parent = msg
child = msg
if full and msg:
# If we have get all the messages, replace NotLoaded with None as
# parent.
msg.parent = None
if not full and not msg:
# Perhaps there are hidden messages
msg = NotLoaded
thread.root = msg
return thread
def iter_unread_messages(self, thread=None):
try:
contacts = {}
with self.browser:
threads = self.browser.get_threads_list()
for thread in threads:
if thread['member'].get('isBan', True):
with self.browser:
self.browser.delete_thread(int(thread['member']['id']))
continue
if self.antispam and not self.antispam.check_thread(thread):
self.logger.info('Skipped a spam-unread-thread from %s' % thread['member']['pseudo'])
self.report_spam(thread['member']['id'])
continue
slut = self._get_slut(thread['member']['id'])
if parse_dt(thread['date']) > slut['lastmsg'] or int(thread['status']) != int(slut['status']):
t = self.get_thread(thread['member']['id'], contacts, get_profiles=True)
for m in t.iter_all_messages():
if m.flags & m.IS_UNREAD:
yield m
if not self.config['baskets'].get():
return
# Send mail when someone added me in her basket.
# XXX possibly race condition if a slut adds me in her basket
# between the aum.nb_new_baskets() and aum.get_baskets().
with self.browser:
slut = self._get_slut(-self.MAGIC_ID_BASKET)
new_baskets = self.browser.nb_new_baskets()
if new_baskets > 0:
baskets = self.browser.get_baskets()
my_name = self.browser.get_my_name()
for basket in baskets:
if basket['isBan'] or parse_dt(basket['date']) <= slut['lastmsg']:
continue
contact = self.get_contact(basket['id'])
if self.antispam and not self.antispam.check_contact(contact):
self.logger.info('Skipped a spam-basket from %s' % contact.name)
self.report_spam(basket['id'])
continue
thread = Thread(int(basket['id']))
thread.title = 'Basket of %s' % contact.name
thread.root = Message(thread=thread,
id=self.MAGIC_ID_BASKET,
title=thread.title,
sender=contact.name,
receivers=[my_name],
date=parse_dt(basket['date']),
content='You are taken in her basket!',
signature=contact.get_text(),
children=[],
flags=Message.IS_UNREAD)
yield thread.root
except BrowserUnavailable, e:
self.logger.debug('No messages, browser is unavailable: %s' % e)
pass # don't care about waiting
def set_message_read(self, message):
if message.id == self.MAGIC_ID_BASKET:
# Save the last baskets checks.
slut = self._get_slut(-self.MAGIC_ID_BASKET)
if slut['lastmsg'] < message.date:
slut['lastmsg'] = message.date
self.storage.set('sluts', -self.MAGIC_ID_BASKET, slut)
self.storage.save()
return
slut = self._get_slut(message.thread.id)
if slut['lastmsg'] < message.date:
slut['lastmsg'] = message.date
self.storage.set('sluts', message.thread.id, slut)
self.storage.save()
def _get_slut(self, id):
id = int(id)
sluts = self.storage.get('sluts')
if not sluts or not id in sluts:
slut = {'lastmsg': datetime.datetime(1970,1,1),
'status': 0}
else:
slut = self.storage.get('sluts', id)
slut['lastmsg'] = slut.get('lastmsg', datetime.datetime(1970,1,1)).replace(tzinfo=tz.tzutc())
slut['status'] = int(slut.get('status', 0))
return slut
# ---- ICapMessagesPost methods ---------------------
def post_message(self, message):
with self.browser:
self.browser.post_mail(message.thread.id, message.content)
# ---- ICapContact methods ---------------------
def fill_contact(self, contact, fields):
if 'profile' in fields:
contact = self.get_contact(contact)
if contact and 'photos' in fields:
for name, photo in contact.photos.iteritems():
with self.browser:
if photo.url and not photo.data:
data = self.browser.openurl(photo.url).read()
contact.set_photo(name, data=data)
if photo.thumbnail_url and not photo.thumbnail_data:
data = self.browser.openurl(photo.thumbnail_url).read()
contact.set_photo(name, thumbnail_data=data)
def fill_photo(self, photo, fields):
with self.browser:
if 'data' in fields and photo.url and not photo.data:
photo.data = self.browser.readurl(photo.url)
if 'thumbnail_data' in fields and photo.thumbnail_url and not photo.thumbnail_data:
photo.thumbnail_data = self.browser.readurl(photo.thumbnail_url)
return photo
def get_contact(self, contact):
with self.browser:
if isinstance(contact, Contact):
_id = contact.id
elif isinstance(contact, (int,long,basestring)):
_id = contact
else:
raise TypeError("The parameter 'contact' isn't a contact nor a int/long/str/unicode: %s" % contact)
profile = self.browser.get_profile(_id)
if not profile:
return None
_id = profile['id']
if isinstance(contact, Contact):
contact.id = _id
contact.name = profile['pseudo']
else:
contact = Contact(_id, profile['pseudo'], Contact.STATUS_ONLINE)
contact.url = self.browser.id2url(_id)
contact.parse_profile(profile, self.browser.get_consts())
return contact
def iter_contacts(self, status=Contact.STATUS_ALL, ids=None):
with self.browser:
threads = self.browser.get_threads_list(count=100)
for thread in threads:
contact = thread['member']
if contact.get('isBan', True):
with self.browser:
self.browser.delete_thread(int(contact['id']))
continue
s = 0
if contact['isOnline']:
s = Contact.STATUS_ONLINE
else:
s = Contact.STATUS_OFFLINE
if not status & s or (ids and not contact['id'] in ids):
continue
c = Contact(contact['id'], contact['pseudo'], s)
c.url = self.browser.id2url(contact['id'])
birthday = _parse_dt(contact['birthday'])
age = int((datetime.datetime.now() - birthday).days / 365.25)
c.status_msg = u'%s old' % age
if int(contact['cover']) > 0:
url = 'http://s%s.adopteunmec.com/%s%%(type)s%s.jpg' % (contact['shard'], contact['path'], contact['cover'])
else:
url = 'http://s.adopteunmec.com/www/img/thumb0.gif'
c.set_photo('image%s' % contact['cover'],
url=url % {'type': 'image'},
thumbnail_url=url % {'type': 'thumb0_'})
yield c
def send_query(self, id):
if isinstance(id, Contact):
id = id.id
queries_queue = None
try:
queries_queue = self.get_optimization('QUERIES_QUEUE')
except OptimizationNotFound:
pass
if queries_queue and queries_queue.is_running():
if queries_queue.enqueue_query(id):
return Query(id, 'A charm has been sent')
else:
return Query(id, 'Unable to send charm: it has been enqueued')
else:
with self.browser:
if not self.browser.send_charm(id):
raise QueryError('No enough charms available')
return Query(id, 'A charm has been sent')
# ---- ICapChat methods ---------------------
def iter_chat_messages(self, _id=None):
with self.browser:
return self.browser.iter_chat_messages(_id)
def send_chat_message(self, _id, message):
with self.browser:
return self.browser.send_chat_message(_id, message)
#def start_chat_polling(self):
#self._profile_walker = ProfilesWalker(self.weboob.scheduler, self.storage, self.browser)
# ---- ICapAccount methods ---------------------
ACCOUNT_REGISTER_PROPERTIES = ValuesDict(
Value('username', label='Email address', regexp='^[^ ]+@[^ ]+\.[^ ]+$'),
Value('password', label='Password', regexp='^[^ ]+$', masked=True),
Value('sex', label='Sex', choices={'m': 'Male', 'f': 'Female'}),
Value('birthday', label='Birthday (dd/mm/yyyy)', regexp='^\d+/\d+/\d+$'),
Value('zipcode', label='Zipcode'),
Value('country', label='Country', choices={'fr': 'France', 'be': 'Belgique', 'ch': 'Suisse', 'ca': 'Canada'}, default='fr'),
Value('godfather',label='Godfather', regexp='^\d*$', default=''),
)
@classmethod
def register_account(klass, account):
"""
Register an account on website
This is a static method, it would be called even if the backend is
instancied.
@param account an Account object which describe the account to create
"""
browser = None
bday, bmonth, byear = account.properties['birthday'].get().split('/', 2)
while not browser:
try:
browser = klass.BROWSER(account.properties['username'].get())
browser.register(password= account.properties['password'].get(),
sex= (0 if account.properties['sex'].get() == 'm' else 1),
birthday_d= int(bday),
birthday_m= int(bmonth),
birthday_y= int(byear),
zipcode= account.properties['zipcode'].get(),
country= account.properties['country'].get(),
godfather= account.properties['godfather'].get())
except CaptchaError:
getLogger('aum').info('Unable to resolve captcha. Retrying...')
browser = None
REGISTER_REGEXP = re.compile('.*http://www.adopteunmec.com/register4.php\?([^\' ]*)\'')
def confirm_account(self, mail):
msg = email.message_from_string(mail)
content = u''
for part in msg.walk():
s = part.get_payload(decode=True)
content += unicode(s, 'iso-8859-15')
url = None
for s in content.split():
m = self.REGISTER_REGEXP.match(s)
if m:
url = '/register4.php?' + m.group(1)
break
if url:
browser = self.create_browser('')
browser.openurl(url)
return True
return False
def get_account(self):
"""
Get the current account.
"""
raise NotImplementedError()
def update_account(self, account):
"""
Update the current account.
"""
raise NotImplementedError()
def get_account_status(self):
with self.browser:
return (
StatusField('myname', 'My name', self.browser.get_my_name()),
StatusField('score', 'Score', self.browser.score()),
StatusField('avcharms', 'Available charms', self.browser.nb_available_charms()),
StatusField('godchilds', 'Number of godchilds', self.browser.nb_godchilds()),
)
OBJECTS = {Thread: fill_thread,
Contact: fill_contact,
ContactPhoto: fill_photo
}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/browser.py 0000664 0000000 0000000 00000032034 11666415431 0026520 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2008-2011 Romain Bignon, Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import math
import re
import datetime
import random
import urllib
from htmlentitydefs import codepoint2name
try:
import json
except ImportError:
import simplejson as json
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword, BrowserUnavailable
from weboob.capabilities.chat import ChatException, ChatMessage
from weboob.capabilities.messages import CantSendMessage
__all__ = ['AuMBrowser']
class AuMException(Exception):
ERRORS = {"0.0.0": "Bad signature",
"0.0.1": "Malformed request",
"0.0.2": "Not logged",
"1.1.1": "No member has this login",
"1.1.2": "Password don't match",
"1.1.3": "User has been banned",
"1.12.1": "Invalid country",
"1.12.1": "Invalid region",
"4.1.1": "Thread doesn't exist",
"4.1.2": "Cannot write to this member",
"5.1.1": "Member tergeted doesn't exist",
"5.1.2": "Sex member targeted is not the opposite of the member logged",
"5.1.3": "Not possible to send a charm",
"5.1.4": "Not possible to send a charm because the 5 charms has been already used",
"5.1.5": "Not possible because the guy has already send a charm to this girl",
"5.1.6": "No more money",
"5.1.7": "Not possible to add to basket",
"5.2.1": "Member doesn't exist",
"5.3.1": "Member doesn't exist",
}
def __init__(self, code):
Exception.__init__(self, self.ERRORS.get(code, code))
self.code = code
class AuMBrowser(BaseBrowser):
DOMAIN = 'api.adopteunmec.com'
APIKEY = 'fb0123456789abcd'
consts = None
search_query = None
my_sex = 0
my_id = 0
my_name = u''
my_coords = (0,0)
def id2url(self, id):
return 'http://www.adopteunmec.com/%s' % id
def api_request(self, command, action, parameter='', data=None, nologin=False):
if data is None:
# Always do POST requests.
data = ''
elif isinstance(data, (list,tuple,dict)):
data = urllib.urlencode(data)
elif isinstance(data, unicode):
data = data.encode('utf-8')
url = self.buildurl(self.absurl('/api.php'), S=self.APIKEY,
C=command,
A=action,
P=parameter,
O='json')
buf = self.openurl(url, data)
try:
r = json.load(buf)
except ValueError:
buf.seek(0)
raise ValueError(buf.read())
if 'errors' in r and r['errors'] != '0' and len(r['errors']) > 0:
code = r['errors'][0]
if code in (u'0.0.2', u'1.1.1', u'1.1.2'):
if not nologin:
self.login()
return self.api_request(command, action, parameter, data, nologin=True)
else:
raise BrowserIncorrectPassword(AuMException.ERRORS[code])
else:
raise AuMException(code)
return r
def login(self):
r = self.api_request('me', 'login', data={'login': self.username,
'pass': self.password,
}, nologin=True)
self.my_sex = r['result']['me']['sex']
self.my_id = int(r['result']['me']['id'])
self.my_name = r['result']['me']['pseudo']
self.my_coords = (float(r['result']['me']['lat']), float(r['result']['me']['lng']))
return r
#def register(self, password, sex, birthday_d, birthday_m, birthday_y, zipcode, country, godfather=None):
# if not self.is_on_page(RegisterPage):
# self.location('http://www.adopteunmec.com/register2.php')
# self.page.register(password, sex, birthday_d, birthday_m, birthday_y, zipcode, country)
# if godfather:
# if not self.is_on_page(AccountPage):
# self.location('http://www.adopteunmec.com/account.php')
# self.page.set_godfather(godfather)
#@pageaccess
#def add_photo(self, name, f):
# if not self.is_on_page(EditPhotoPage):
# self.location('/edit.php?type=1')
# return self.page.add_photo(name, f)
#@pageaccess
#def set_nickname(self, nickname):
# if not self.is_on_page(EditAnnouncePage):
# self.location('/edit.php?type=2')
# return self.page.set_nickname(nickname)
#@pageaccess
#def set_announce(self, title=None, description=None, lookingfor=None):
# if not self.is_on_page(EditAnnouncePage):
# self.location('/edit.php?type=2')
# return self.page.set_announce(title, description, lookingfor)
#@pageaccess
#def set_description(self, **args):
# if not self.is_on_page(EditDescriptionPage):
# self.location('/edit.php?type=3')
# return self.page.set_description(**args)
def check_login(func):
def inner(self, *args, **kwargs):
if self.my_id == 0:
self.login()
return func(self, *args, **kwargs)
return inner
def get_consts(self):
if self.consts is not None:
return self.consts
self.consts = []
for i in xrange(2):
r = self.api_request('me', 'all_values', data={'sex': i})
self.consts.append(r['result']['values'])
return self.consts
@check_login
def score(self):
r = self.api_request('member', 'view', data={'id': self.my_id})
return int(r['result']['member']['popu']['popu'])
@check_login
def get_my_name(self):
return self.my_name
@check_login
def get_my_id(self):
return self.my_id
@check_login
def nb_new_mails(self):
r = self.api_request('me', '[default]')
return r['result']['news']['newMails']
@check_login
def nb_new_baskets(self):
r = self.api_request('me', '[default]')
return r['result']['news']['newBaskets']
@check_login
def nb_new_visites(self):
r = self.api_request('me', '[default]')
return r['result']['news']['newVisits']
@check_login
def nb_available_charms(self):
r = self.login()
return r['result']['flashs']
@check_login
def nb_godchilds(self):
r = self.api_request('member', 'view', data={'id': self.my_id})
return int(r['result']['member']['popu']['invits'])
@check_login
def get_baskets(self):
r = self.api_request('me', 'basket')
return r['result']['basket']
@check_login
def get_threads_list(self, count=30):
r = self.api_request('message', '[default]', '%d,0' % count)
return r['result']['threads']
@check_login
def get_thread_mails(self, id, count=30):
r = self.api_request('message', 'thread', data={'memberId': id, 'count': count})
return r['result']['thread']
@check_login
def post_mail(self, id, content):
new_content = u''
for c in content:
try:
new_content += '&%s;' % codepoint2name[ord(c)]
except KeyError:
new_content += c
content = new_content.replace('\n', '\r\n').encode('Windows-1252', 'replace')
try:
self.api_request('message', 'new', data={'memberId': id, 'message': content})
except AuMException, e:
raise CantSendMessage(unicode(e))
@check_login
def delete_thread(self, id):
r = self.api_request('message', 'delete', data={'id_user': id})
self.logger.debug('Thread deleted: %r' % r)
@check_login
def send_charm(self, id):
try:
self.api_request('member', 'addBasket', data={'id': id})
except AuMException:
return False
else:
return True
@check_login
def add_basket(self, id):
try:
self.api_request('member', 'addBasket', data={'id': id})
except AuMException:
return False
else:
return True
def deblock(self, id):
self.readurl('http://www.adopteunmec.com/fajax_postMessage.php?action=deblock&to=%s' % id)
return True
def report_fake(self, id):
return self.readurl('http://www.adopteunmec.com/fake.php', 'id=%s' % id)
def rate(self, id, what, rating):
result = self.openurl('http://www.adopteunmec.com/fajax_vote.php', 'member=%s&what=%s&rating=%s' % (id, what, rating)).read()
return float(result)
def search_profiles(self, **kwargs):
if self.search_query is None:
r = self.api_request('searchs', '[default]')
self.search_query = r['result']['search']['query']
params = json.loads(self.search_query)
r = self.api_request('searchs', 'advanced', '30,0', params)
ids = [s['id'] for s in r['result']['search']]
return set(ids)
def get_profile(self, id, with_pics=True):
r = self.api_request('member', 'view', data={'id': id})
profile = r['result']['member']
# Calculate distance in km.
coords = (float(profile['lat']), float(profile['lng']))
R = 6371
lat1 = math.radians(self.my_coords[0])
lat2 = math.radians(coords[0])
lon1 = math.radians(self.my_coords[1])
lon2 = math.radians(coords[1])
dLat = lat2 - lat1
dLong = lon2 - lon1
a= pow(math.sin(dLat/2), 2) + math.cos(lat1) * math.cos(lat2) * pow(math.sin(dLong/2), 2)
c= 2 * math.atan2(math.sqrt(a), math.sqrt(1-a))
profile['dist'] = R * c
if with_pics:
r = self.api_request('member', 'pictures', data={'id': id})
profile['pictures'] = []
for pic in r['result']['pictures']:
d = {'hidden': False}
d.update(pic)
profile['pictures'].append(d)
if len(profile['pictures']) > 0:
pic_regex = re.compile('(?Phttp://.+\.adopteunmec\.com/.+/)image(?P.+)\.jpg')
pic_max_id = max(int(pic_regex.match(pic['url']).groupdict()['id']) for pic in profile['pictures'])
base_url = pic_regex.match(profile['pictures'][0]['url']).groupdict()['base_url']
for id in xrange(1, pic_max_id + 1):
url = u'%simage%s.jpg' % (base_url, id)
if not url in [pic['url'] for pic in profile['pictures']]:
profile['pictures'].append({'url': url, u'hidden': True, 'id': u'0', 'rating': 0.0})
return profile
def _get_chat_infos(self):
try:
data = json.load(self.openurl('http://www.adopteunmec.com/1.1_cht_get.php?anticache=%f' % random.random()))
except ValueError:
raise BrowserUnavailable()
if data['error']:
raise ChatException(u'Error while getting chat infos. json:\n%s' % data)
return data
def iter_contacts(self):
def iter_dedupe(contacts):
yielded_ids = set()
for contact in contacts:
if contact['id'] not in yielded_ids:
yield contact
yielded_ids.add(contact['id'])
data = self._get_chat_infos()
return iter_dedupe(data['contacts'])
def iter_chat_messages(self, _id=None):
data = self._get_chat_infos()
if data['messages'] is not None:
for message in data['messages']:
yield ChatMessage(id_from=message['id_from'], id_to=message['id_to'], message=message['message'], date=message['date'])
def send_chat_message(self, _id, message):
url = 'http://www.adopteunmec.com/1.1_cht_send.php?anticache=%f' % random.random()
data = dict(id=_id, message=message)
headers = {
'Content-type': 'application/x-www-form-urlencoded',
'Accept': 'text/plain',
'Referer': 'http://www.adopteunmec.com/chat.php',
'Origin': 'http://www.adopteunmec.com',
}
request = self.request_class(url, urllib.urlencode(data), headers)
response = self.openurl(request).read()
try:
datetime.datetime.strptime(response, '%Y-%m-%d %H:%M:%S')
return True
except ValueError:
return False
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/captcha.py 0000664 0000000 0000000 00000014767 11666415431 0026455 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import hashlib
import sys
import Image
class CaptchaError(Exception): pass
class Tile:
hash = {
'bc8d52d96058478a6def26226145d53b': 'A',
'c62ecdfddb72b2feaed96cd9fe7c2802': 'A',
'8b61cda8a3240d8fa5f2610424271300': 'AD',
'f5dc63d37c7ea3375d86180f0ae62d05': 'AE',
'fd562be230da7f767f4454148632201d': 'AF',
'1860de576d8b0d1d87edc9dcb0b2a64c': 'AG',
'53afa108d36186e6bd23631711ec3d8c': 'AJ',
'6f2f9a1082a9230272c45117320f159d': 'AL',
'e14249a774d24bacc6e2bcadd7f3df65': 'AM',
'389330dbf3d85dea2dc40c6f9cf77d52': 'AN',
'17526a3c2261b55f9cd237c4aa195099': 'AQ',
'7e4820a9cc6c83a9fa60ff73ceb52157': 'AW',
'90690d1209753a2bcfeafa890082a585': 'B',
'2cf22e9ceace03a5f8ed3999e92d877e': 'C',
'a1d0bf1a29600a82a6aa2b8b21651b0f': 'D',
'9bb6909d647a0be3b2e7352d37374228': 'E',
'38120c8346f16cd07a9194283787ee5e': 'F',
'd41ff948fbc50a628c858b8e3e9e931c': 'G',
'4cc9322d3361eb3f9fea7fc83579e40f': 'H',
'837cd0f04e2d47ca6975745bdd0da640': 'I',
'da0204fa51b38414051376cc1c27ba72': 'J',
'199b1a9f9e1df1c2eddadcc4582957d7': 'JW',
'5e8d3d5bd5f683d84b089f2cecc1e196': 'JX',
'bc1fcf3546057d40d2db5454caacb3a5': 'JZ',
'c2f5866ba3bf799ece8b202492d199bf': 'K',
'7abe4091e11921afe6dac16509999010': 'KT',
'281ef08e623184e5621a73b9ccec7c9a': 'KX',
'b28e3fc06411de2ac7f53569bc3b42db': 'L',
'd58a6c26649926f1145fb4b7b42d0554': 'LT',
'4add630c6d124899fef814211975e344': 'M',
'9740cefe1629d6bc149a72d5f2a4586d': 'N',
'396f816f7e78e5c98de6404f8c4bd2ee': 'O',
'31ae7c9536b6c6a96e30a77b70e4b2fd': 'P',
'98ad9b1c32c05e6efc06637a166e4c42': 'PA',
'a05cce33683025fb2c6708ee06f6028e': 'Q',
'2852f51e8939bf9664fe064f7dacf310': 'R',
'3798513fe87e786faa67552a140fd86f': 'S',
'350b13811e34eeb63e3d7fb4b5eade5b': 'T',
'a01b186cbc767e17d948ed04eff114a1': 'U',
'8405f4d80ce80c4e6e9680fcfac4fe40': 'V',
'17ed80e9cb9a585098ae6a55d8d1f5c0': 'W',
'ae54ca77be5561330781a08dfbaff7a7': 'W',
'bbded6a2ba5f521bba276bb843bf4c98': 'WXT',
'ea662dd25fc528b84b832ce71ae3de61': 'WZ',
'4eb23916138e7c01714431dbecfe8b96': 'X',
'c02093d35d852339ff34f2b26873bf5a': 'XW',
'65744e0c6ce0c56d04873dfd732533a7': 'Y',
'315fb7dba7032004bd362cf0bb076733': 'YA',
'ce12a68a4f15657bc5297a6cf698bc0a': 'YAQ',
'275478ea2280351f7433a0606f962175': 'Z',
}
def __init__(self):
self.map = []
def append(self, pxls):
self.map.append(pxls)
def display(self):
print '-' * (len(self.map) * 2 + 2)
for y in xrange(len(self.map[0])):
sys.stdout.write('|')
for x in xrange(len(self.map)):
sys.stdout.write('%s' % ('XX' if self.map[x][y] else ' '))
print '|'
print '-' * (len(self.map) * 2 + 2)
def checksum(self):
s = ''
for pxls in self.map:
for pxl in pxls:
s += '%d' % (1 if pxl else 0)
return hashlib.md5(s).hexdigest()
@property
def letter(self):
checksum = self.checksum()
try:
return self.hash[checksum]
except KeyError:
print 'Unable te resolve:'
self.display()
print 'hash: %s' % checksum
raise CaptchaError()
class Captcha:
def __init__(self, f):
self.img = Image.open(f)
self.w, self.h = self.img.size
self.map = self.img.load()
self.tiles = []
tile = None
for x in xrange(self.w):
blank = True
pxls = []
for y in xrange(self.h):
pxls.append(self[x,y])
if self[x,y] != 0:
blank = False
if tile:
if blank:
tile = None
else:
tile.append(pxls)
elif not blank:
tile = Tile()
tile.append(pxls)
self.tiles.append(tile)
def __getitem__(self, (x, y)):
return self.map[x % self.w, y % self.h]
def __iter__(self):
for tile in self.tiles:
yield tile
@property
def text(self):
s = ''
for tile in self.tiles:
s += tile.letter
return s
class Decoder:
def __init__(self):
self.hash = {}
def process(self):
from weboob.backends.aum.browser import AuMBrowser
browser = AuMBrowser('')
browser.openurl('/register2.php')
c = Captcha(browser.openurl('/captcha.php'))
for tile in c:
checksum = tile.checksum()
if checksum in self.hash:
print 'Skipping %s' % self.hash[checksum]
continue
tile.display()
print 'Checksum: %s' % checksum
ntry = 2
while ntry:
sys.stdout.write('Enter the letter: ')
l = sys.stdin.readline().strip()
ntry -= 1
if len(l) != 1:
print 'Error: please enter only one letter'
elif l in self.hash.itervalues():
print 'Warning! This letter has already been catched!'
else:
ntry = 0
self.hash[checksum] = l
def main(self):
try:
while 1:
self.process()
except KeyboardInterrupt:
print ''
print 'hash = {'
l = sorted(self.hash.iteritems(), key=lambda (k,v): (v,k))
for hash, value in l:
print ' \'%s\': %s' % (hash, value)
print '}'
if __name__ == '__main__':
d = Decoder()
d.main()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/contact.py 0000664 0000000 0000000 00000027010 11666415431 0026466 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2008-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import socket
from datetime import datetime
from dateutil.parser import parse as parse_dt
from weboob.tools.ordereddict import OrderedDict
from weboob.capabilities.contact import Contact as _Contact, ProfileNode
from weboob.tools.misc import html2text
class FieldBase:
def __init__(self, key, key2=None):
self.key = key
self.key2 = key2
def get_value(self, value, consts):
raise NotImplementedError
class FieldStr(FieldBase):
def get_value(self, profile, consts):
return html2text(unicode(profile[self.key])).strip()
class FieldBool(FieldBase):
def get_value(self, profile, consts):
return bool(int(profile[self.key]))
class FieldDist(FieldBase):
def get_value(self, profile, consts):
return '%.2f km' % float(profile[self.key])
class FieldIP(FieldBase):
def get_hostname(self, s):
try:
return socket.gethostbyaddr(s)[0]
except (socket.gaierror, socket.herror):
return s
def get_value(self, profile, consts):
s = self.get_hostname(profile[self.key])
if profile[self.key] != profile[self.key2]:
s += ' (first %s)' % self.get_hostname(profile[self.key2])
return s
class FieldProfileURL(FieldBase):
def get_value(self, profile, consts):
id = int(profile[self.key])
if id > 0:
return 'http://www.adopteunmec.com/%d' % id
else:
return ''
class FieldPopu(FieldBase):
def get_value(self, profile, consts):
return unicode(profile['popu'][self.key])
class FieldOld(FieldBase):
def get_value(self, profile, consts):
birthday = parse_dt(profile[self.key])
return int((datetime.now() - birthday).days / 365.25)
class FieldSplit(FieldBase):
def get_value(self, profile, consts):
return [html2text(s).strip() for s in profile[self.key].split(self.key2) if len(s.strip()) > 0]
class FieldBMI(FieldBase):
def __init__(self, key, key2, fat=False):
FieldBase.__init__(self, key, key2)
self.fat = fat
def get_value(self, profile, consts):
height = int(profile[self.key])
weight = int(profile[self.key2])
if height == 0 or weight == 0:
return ''
bmi = (weight/float(pow(height/100.0, 2)))
if not self.fat:
return bmi
elif bmi < 15.5:
return 'severely underweight'
elif bmi < 18.4:
return 'underweight'
elif bmi < 24.9:
return 'normal'
elif bmi < 30:
return 'overweight'
else:
return 'obese'
class FieldFlags(FieldBase):
def get_value(self, profile, consts):
i = int(profile[self.key])
labels = []
for d in consts[self.key]:
if i & (1 << int(d['value'])):
labels.append(html2text(d['label']).strip())
return labels
class FieldList(FieldBase):
def get_value(self, profile, consts):
i = int(profile[self.key])
for d in consts[self.key]:
if i == int(d['value']):
return html2text(d['label']).strip()
return ''
class Contact(_Contact):
TABLE = OrderedDict((
('_info', OrderedDict((
('IPaddr', FieldIP('last_ip', 'first_ip')),
('admin', FieldBool('admin')),
('ban', FieldBool('isBan')),
('first', FieldStr('first_cnx')),
('godfather', FieldProfileURL('godfather')),
))),
('_stats', OrderedDict((
('mails', FieldPopu('mails')),
('baskets', FieldPopu('contacts')),
('charms', FieldPopu('flashs')),
('visites', FieldPopu('visites')),
('invits', FieldPopu('invits')),
('bonus', FieldPopu('bonus')),
('score', FieldPopu('popu')),
))),
('details', OrderedDict((
('old', FieldOld('birthday')),
('birthday', FieldStr('birthday')),
('zipcode', FieldStr('zip')),
('location', FieldStr('city')),
('distance', FieldDist('dist')),
('country', FieldStr('country')),
('eyes', FieldList('eyes')),
('hair_color', FieldList('hair_color')),
('hair_size', FieldList('hair_size')),
('height', FieldList('size')),
('weight', FieldList('weight')),
('BMI', FieldBMI('size', 'weight')),
('fat', FieldBMI('size', 'weight', fat=True)),
('origins', FieldList('origins')),
('signs', FieldFlags('checks1')),
('job', FieldStr('job')),
('style', FieldList('style')),
('food', FieldList('food')),
('drink', FieldList('drink')),
('smoke', FieldList('smoke')),
))),
('tastes', OrderedDict((
('hobbies', FieldStr('hobbies')),
('music', FieldSplit('music', ' ')),
('cinema', FieldSplit('cinema', ' ')),
('books', FieldSplit('books', ' ')),
('tv', FieldSplit('tvs', ' ')),
))),
('sex', OrderedDict((
('underwear', FieldFlags('checks7')),
('practices', FieldFlags('checks5')),
('favorite', FieldFlags('checks3')),
('toys', FieldFlags('checks6')),
))),
('personality', OrderedDict((
('snap', FieldStr('texts1')),
('exciting', FieldStr('texts2')),
('hate', FieldStr('texts3')),
('vices', FieldStr('texts4')),
('assets', FieldStr('texts6')),
('fantasies', FieldStr('texts5')),
('is', FieldFlags('checks2')),
)))
))
def parse_profile(self, profile, consts):
if int(profile['cat']) == 1:
self.status = Contact.STATUS_ONLINE
self.status_msg = u'online'
self.status_msg = u'since %s' % profile['last_cnx']
elif int(profile['cat']) == 2:
self.status = Contact.STATUS_AWAY
self.status_msg = u'away'
self.status_msg = u'connection at %s' % profile['last_cnx']
elif int(profile['cat']) == 3:
self.status = Contact.STATUS_OFFLINE
self.status_msg = u'last connection %s' % profile['last_cnx']
self.summary = html2text(profile['about1']).strip().replace('\n\n', '\n')
if len(profile['about2']) > 0:
self.summary += u'\n\nLooking for:\n%s' % html2text(profile['about2']).strip().replace('\n\n', '\n')
for photo in profile['pictures']:
self.set_photo(photo['url'].split('/')[-1],
url=photo['url'],
thumbnail_url=photo['url'].replace('image', 'thumb1_'),
hidden=photo['hidden'])
self.profile = OrderedDict()
for section, d in self.TABLE.iteritems():
flags = ProfileNode.SECTION
if section.startswith('_'):
flags |= ProfileNode.HEAD
section = section.lstrip('_')
s = ProfileNode(section, section.capitalize(), OrderedDict(), flags=flags)
for key, builder in d.iteritems():
value = builder.get_value(profile, consts[int(profile['sex'])])
s.value[key] = ProfileNode(key, key.capitalize().replace('_', ' '), value)
self.profile[section] = s
self.aum_profile = profile
def get_text(self):
def print_node(node, level=1):
result = u''
if node.flags & node.SECTION:
result += u'\t' * level + node.label + '\n'
for sub in node.value.itervalues():
result += print_node(sub, level+1)
else:
if isinstance(node.value, (tuple,list)):
value = ', '.join(unicode(v) for v in node.value)
elif isinstance(node.value, float):
value = '%.2f' % node.value
else:
value = node.value
result += u'\t' * level + u'%-20s %s\n' % (node.label + ':', value)
return result
result = u'Nickname: %s\n' % self.name
if self.status & Contact.STATUS_ONLINE:
s = 'online'
elif self.status & Contact.STATUS_OFFLINE:
s = 'offline'
elif self.status & Contact.STATUS_AWAY:
s = 'away'
else:
s = 'unknown'
result += u'Status: %s (%s)\n' % (s, self.status_msg)
result += u'URL: %s\n' % self.url
result += u'Photos:\n'
for name, photo in self.photos.iteritems():
result += u'\t%s%s\n' % (photo, ' (hidden)' if photo.hidden else '')
result += u'\nProfile:\n'
for head in self.profile.itervalues():
result += print_node(head)
result += u'Description:\n'
for s in self.summary.split('\n'):
result += u'\t%s\n' % s
return result
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/optim/ 0000775 0000000 0000000 00000000000 11666415431 0025611 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/optim/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0027710 0 ustar 00root root 0000000 0000000 priority_connection.py 0000664 0000000 0000000 00000015317 11666415431 0032213 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/optim # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
import random
from weboob.tools.browser import BrowserUnavailable, BrowserIncorrectPassword
from weboob.capabilities.dating import Optimization
from weboob.capabilities.account import AccountRegisterError
from weboob.tools.log import getLogger
from weboob.tools.value import Value, ValuesDict, ValueInt
from weboob.backends.aum.captcha import CaptchaError
from weboob.backends.aum.exceptions import AdopteWait, AdopteBanned
from weboob.backends.aum.browser import AuMBrowser
__all__ = ['PriorityConnection']
class PriorityConnection(Optimization):
CONFIG = ValuesDict(ValueInt('minimal', label='Minimal of godchilds', default=5),
Value('domain', label='Domain to use for fake accounts emails', default='aum.example.com'),
ValueInt('interval', label='Interval of checks (seconds)', default=3600)
)
def __init__(self, sched, storage, browser):
self.sched = sched
self.storage = storage
self.browser = browser
self.logger = getLogger('priorityconn', browser.logger)
self.config = storage.get('priority_connection', 'config', default=None)
if self.config == {}:
self.config = None
self.check_cron = None
self.activity_cron = None
def start(self):
if self.config is None:
return False
self.check_cron = self.sched.repeat(int(self.config['interval']), self.check_godchilds)
self.activity_cron = self.sched.repeat(600, self.activity_fakes)
return True
def stop(self):
self.sched.cancel(self.check_cron)
self.check_cron = None
self.sched.cancel(self.activity_cron)
self.activity_cron = None
return True
def is_running(self):
return self.check_cron is not None
def set_config(self, params):
self.config = params
self.storage.set('priority_connection', 'config', self.config)
self.storage.save()
def get_config(self):
return self.config
def generate_name(self):
login = u''
for x in xrange(8):
if x % 2:
login += random.choice(u'aeiou')
else:
login += random.choice(u'bcdfghjklmnprstv')
fakes = self.storage.get('priority_connection', 'fakes')
while ('%s@%s' % (login, self.config['domain'])) in fakes.iterkeys():
login += '_'
return login
def generate_password(self):
return '%08x' % random.randint(1, int('ffffffff', 16))
def check_godchilds(self):
with self.browser:
try:
my_id = self.browser.get_my_id()
nb_godchilds = self.browser.nb_godchilds()
except AdopteWait:
nb_godchilds = 0
except BrowserUnavailable:
# We'll check later
return
missing_godchilds = int(self.config['minimal']) - nb_godchilds
self.logger.info('Missing godchilds: %s' % missing_godchilds)
if missing_godchilds <= 0:
return
for i in xrange(missing_godchilds):
registered = False
while not registered:
name = self.generate_name()
password = self.generate_password()
browser = AuMBrowser('%s@%s' % (name, self.config['domain']), proxy=self.browser.proxy)
try:
browser.register(password= password,
sex= 1, #slut
birthday_d= random.randint(1,28),
birthday_m= random.randint(1,12),
birthday_y= random.randint(1975, 1990),
zipcode= 75001,
country= 'fr',
godfather= my_id)
except AccountRegisterError, e:
self.logger.warning('Unable to register account: %s' % e)
except CaptchaError:
self.logger.warning('Unable to solve captcha... Retrying')
else:
registered = True
# set nickname
browser.set_nickname(name.strip('_').capitalize())
# rate my own profile with good score
for i in xrange(4):
browser.rate(my_id, i, 5.0)
# save fake in storage
fake = {'username': browser.username,
'password': password}
self.storage.set('priority_connection', 'fakes', name, fake)
self.storage.save()
self.logger.info('Fake account "%s" created (godfather=%s)' % (name, my_id))
def activity_fakes(self):
try:
fakes = self.storage.get('priority_connection', 'fakes', default={})
if len(fakes) == 0:
return
while 1:
name = random.choice(fakes.keys())
fake = fakes[name]
try:
browser = AuMBrowser(fake['username'], fake['password'], proxy=self.browser.proxy)
except (AdopteBanned,BrowserIncorrectPassword), e:
self.logger.warning('Fake %s can\'t login: %s' % (name, e))
continue
profiles = browser.search_profiles(country="fr",
dist='10',
save=True)
if not profiles:
continue
id = profiles.pop()
profile = browser.get_profile(id)
# bad rate
for i in xrange(4):
browser.rate(profile.get_id(), i, 0.6)
# deblock
browser.deblock(profile.get_id())
return
except BrowserUnavailable:
# don't care
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/optim/profiles_walker.py 0000664 0000000 0000000 00000007157 11666415431 0031365 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from random import randint
from weboob.tools.browser import BrowserUnavailable
from weboob.capabilities.dating import Optimization
from weboob.tools.log import getLogger
__all__ = ['ProfilesWalker']
class ProfilesWalker(Optimization):
def __init__(self, sched, storage, browser):
self.sched = sched
self.storage = storage
self.browser = browser
self.logger = getLogger('walker', browser.logger)
self.walk_cron = None
self.view_cron = None
self.visited_profiles = set(storage.get('profiles_walker', 'viewed'))
self.logger.info(u'Loaded %d already visited profiles from storage.' % len(self.visited_profiles))
self.profiles_queue = set()
def save(self):
self.storage.set('profiles_walker', 'viewed', list(self.visited_profiles))
self.storage.save()
def start(self):
self.walk_cron = self.sched.repeat(60, self.enqueue_profiles)
self.view_cron = self.sched.schedule(randint(10,40), self.view_profile)
return True
def stop(self):
self.sched.cancel(self.walk_cron)
self.sched.cancel(self.view_cron)
self.walk_cron = None
self.view_cron = None
return True
def is_running(self):
return self.walk_cron is not None
def enqueue_profiles(self):
try:
with self.browser:
profiles_to_visit = self.browser.search_profiles().difference(self.visited_profiles)
self.logger.info(u'Enqueuing profiles to visit: %s' % profiles_to_visit)
self.profiles_queue = set(profiles_to_visit)
self.save()
except BrowserUnavailable:
return
def view_profile(self):
try:
try:
id = self.profiles_queue.pop()
except KeyError:
return # empty queue
try:
with self.browser:
profile = self.browser.get_profile(id)
self.logger.info(u'Visited profile %s (%s)' % (profile['pseudo'], id))
# Get score from the aum_score module
#d = self.nucentral_core.callService(context.Context.fromComponent(self), 'aum_score', 'score', profile)
# d.addCallback(self.score_cb, profile.getID())
# deferredlist.append(d)
# do not forget that we visited this profile, to avoid re-visiting it.
self.visited_profiles.add(id)
self.save()
except BrowserUnavailable:
# We consider this profil hasn't been [correctly] analysed
self.profiles_queue.add(id)
return
except Exception, e:
print e
finally:
if self.view_cron is not None:
self.view_cron = self.sched.schedule(randint(10,40), self.view_profile)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/optim/queries_queue.py 0000664 0000000 0000000 00000006551 11666415431 0031053 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.tools.browser import BrowserUnavailable
from weboob.capabilities.dating import Optimization
from weboob.capabilities.contact import QueryError
from weboob.tools.log import getLogger
__all__ = ['QueriesQueue']
class QueriesQueue(Optimization):
def __init__(self, sched, storage, browser):
self.sched = sched
self.storage = storage
self.browser = browser
self.logger = getLogger('queriesqueue', browser.logger)
self.queue = storage.get('queries_queue', 'queue', default=[])
self.check_cron = None
def save(self):
self.storage.set('queries_queue', 'queue', self.queue)
self.storage.save()
def start(self):
self.check_cron = self.sched.repeat(3600, self.flush_queue)
return True
def stop(self):
self.sched.cancel(self.check_cron)
self.check_cron = None
return True
def is_running(self):
return self.check_cron is not None
def enqueue_query(self, id, priority=999):
id_queue = [_id[1] for _id in self.queue]
if int(id) in id_queue:
raise QueryError('This id is already queued')
self.queue.append((int(priority), int(id)))
self.save()
# Try to flush queue to send it now.
self.flush_queue()
# Check if the enqueued query has been sent
for p, i in self.queue:
if i == int(id):
return False
return True
def flush_queue(self):
self.queue.sort()
priority = 0
id = None
try:
try:
while len(self.queue) > 0:
priority, id = self.queue.pop()
if not id:
continue
with self.browser:
if self.browser.send_charm(id):
self.logger.info('Charm sent to %s' % id)
else:
self.queue.append((priority, id))
self.logger.info("Charm can't be send to %s" % id)
break
# As the charm has been correctly sent (no exception raised),
# we don't store anymore ID, because if nbAvailableCharms()
# fails, we don't want to re-queue this ID.
id = None
priority = 0
except BrowserUnavailable:
# We consider this profil hasn't been [correctly] analysed
if not id is None:
self.queue.append((priority, id))
finally:
self.save()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/optim/visibility.py 0000664 0000000 0000000 00000003065 11666415431 0030356 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BrowserUnavailable
from weboob.capabilities.dating import Optimization
from ..browser import AuMBrowser
__all__ = ['Visibility']
class Visibility(Optimization):
def __init__(self, sched, browser):
self.sched = sched
self.browser = browser
self.cron = None
def start(self):
self.cron = self.sched.repeat(60*5, self.reconnect)
return True
def stop(self):
self.sched.cancel(self.cron)
self.cron = None
return True
def is_running(self):
return self.cron is not None
def reconnect(self):
try:
AuMBrowser(self.browser.username,
self.browser.password,
proxy=self.browser.proxy)
except BrowserUnavailable, e:
print str(e)
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/aum/test.py 0000664 0000000 0000000 00000002761 11666415431 0026020 0 ustar 00root root 0000000 0000000 # -*- CODing: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
from weboob.tools.browser import BrowserUnavailable
__all__ = ['AuMTest']
class AuMTest(BackendTest):
BACKEND = 'aum'
def test_new_messages(self):
try:
for message in self.backend.iter_unread_messages():
pass
except BrowserUnavailable:
# enough frequent to do not care about.
pass
def test_contacts(self):
try:
contacts = list(self.backend.iter_contacts())
if len(contacts) == 0:
# so bad, we can't test that...
return
self.backend.fillobj(contacts[0], ['photos', 'profile'])
except BrowserUnavailable:
# enough frequent to do not care about.
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/batoto/ 0000775 0000000 0000000 00000000000 11666415431 0025167 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/batoto/__init__.py 0000664 0000000 0000000 00000001434 11666415431 0027302 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import BatotoBackend
__all__ = ['BatotoBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/batoto/backend.py 0000664 0000000 0000000 00000002506 11666415431 0027133 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderBackend, DisplayPage
__all__ = ['BatotoBackend']
class BatotoBackend(GenericComicReaderBackend):
NAME = 'batoto'
DESCRIPTION = 'Batoto manga reading site'
DOMAIN = 'www.batoto.com'
BROWSER_PARAMS = dict(
img_src_xpath="//img[@id='comic_page']/@src",
page_list_xpath="(//select[@id='page_select'])[1]/option/@value")
ID_REGEXP = r'[^/]+/[^/]+'
URL_REGEXP = r'.+batoto.com/read/_/(%s).+' % ID_REGEXP
ID_TO_URL = 'http://www.batoto.com/read/_/%s'
PAGES = { URL_REGEXP: DisplayPage }
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/batoto/test.py 0000664 0000000 0000000 00000001741 11666415431 0026523 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderTest
class BatotoTest(GenericComicReaderTest):
BACKEND = 'batoto'
def test_download(self):
return self._test_download('26287/yurumates_ch4_by_primitive-scans')
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bnporc/ 0000775 0000000 0000000 00000000000 11666415431 0025162 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bnporc/__init__.py 0000664 0000000 0000000 00000001440 11666415431 0027272 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import BNPorcBackend
__all__ = ['BNPorcBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bnporc/backend.py 0000664 0000000 0000000 00000007437 11666415431 0027136 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
# python2.5 compatibility
from __future__ import with_statement
from weboob.capabilities.bank import ICapBank, AccountNotFound, Account, Recipient
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import ValueBackendPassword
from .browser import BNPorc
__all__ = ['BNPorcBackend']
class BNPorcBackend(BaseBackend, ICapBank):
NAME = 'bnporc'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = 'BNP Paribas french bank\' website'
CONFIG = BackendConfig(ValueBackendPassword('login', label='Account ID', masked=False),
ValueBackendPassword('password', label='Password', regexp='^(\d{6}|)$'),
ValueBackendPassword('rotating_password',
label='Password to set when the allowed uses are exhausted (6 digits)',
regexp='^(\d{6}|)$'))
BROWSER = BNPorc
def create_default_browser(self):
if self.config['rotating_password'].get().isdigit() and len(self.config['rotating_password'].get()) == 6:
rotating_password = self.config['rotating_password'].get()
else:
rotating_password = None
return self.create_browser(self.config['login'].get(),
self.config['password'].get(),
password_changed_cb=self._password_changed_cb,
rotating_password=rotating_password)
def _password_changed_cb(self, old, new):
self.config['password'].set(new)
self.config['rotating_password'].set(old)
self.config.save()
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
if not _id.isdigit():
raise AccountNotFound()
with self.browser:
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
def iter_history(self, account):
with self.browser:
for history in self.browser.get_history(account):
yield history
def iter_operations(self, account):
with self.browser:
for coming in self.browser.get_coming_operations(account):
yield coming
def iter_transfer_recipients(self, ignored):
for account in self.browser.get_transfer_accounts().itervalues():
recipient = Recipient()
recipient.id = account.id
recipient.label = account.label
yield recipient
def transfer(self, account, to, amount, reason=None):
if isinstance(account, Account):
account = account.id
try:
assert account.isdigit()
assert to.isdigit()
amount = float(amount)
except (AssertionError, ValueError):
raise AccountNotFound()
with self.browser:
return self.browser.transfer(account, to, amount, reason)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bnporc/browser.py 0000664 0000000 0000000 00000014045 11666415431 0027223 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import datetime
from logging import warning
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from weboob.capabilities.bank import TransferError, Transfer
from weboob.backends.bnporc import pages
from .errors import PasswordExpired
__all__ = ['BNPorc']
class BNPorc(BaseBrowser):
DOMAIN = 'www.secure.bnpparibas.net'
PROTOCOL = 'https'
ENCODING = None # refer to the HTML encoding
PAGES = {'.*identifiant=DOSSIER_Releves_D_Operation.*': pages.AccountsList,
'.*SAF_ROP.*': pages.AccountHistory,
'.*Action=SAF_CHM.*': pages.ChangePasswordPage,
'.*NS_AVEDT.*': pages.AccountComing,
'.*NS_AVEDP.*': pages.AccountPrelevement,
'.*NS_VIRDF.*': pages.TransferPage,
'.*NS_VIRDC.*': pages.TransferConfirmPage,
'.*/NS_VIRDA\?stp=(?P\d+).*': pages.TransferCompletePage,
'.*Action=DSP_VGLOBALE.*': pages.LoginPage,
'.*type=homeconnex.*': pages.LoginPage,
'.*layout=HomeConnexion.*': pages.ConfirmPage,
'.*SAF_CHM_VALID.*': pages.ConfirmPage,
}
def __init__(self, *args, **kwargs):
self.rotating_password = kwargs.pop('rotating_password', None)
self.password_changed_cb = kwargs.pop('password_changed_cb', None)
BaseBrowser.__init__(self, *args, **kwargs)
def home(self):
self.location('https://www.secure.bnpparibas.net/banque/portail/particulier/HomeConnexion?type=homeconnex')
def is_logged(self):
return not self.is_on_page(pages.LoginPage)
def login(self):
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
assert self.password.isdigit()
if not self.is_on_page(pages.LoginPage):
self.location('https://www.secure.bnpparibas.net/banque/portail/particulier/HomeConnexion?type=homeconnex')
self.page.login(self.username, self.password)
self.location('/NSFR?Action=DSP_VGLOBALE', no_login=True)
if self.is_on_page(pages.LoginPage):
raise BrowserIncorrectPassword()
def change_password(self, new_password):
assert new_password.isdigit() and len(new_password) == 6
self.location('https://www.secure.bnpparibas.net/SAF_CHM?Action=SAF_CHM')
assert self.is_on_page(pages.ChangePasswordPage)
self.page.change_password(self.password, new_password)
if not self.is_on_page(pages.ConfirmPage):
self.logger.error('Oops, unable to change password')
return
self.password, self.rotating_password = (new_password, self.password)
if self.password_changed_cb:
self.password_changed_cb(self.rotating_password, self.password)
def check_expired_password(func):
def inner(self, *args, **kwargs):
try:
return func(self, *args, **kwargs)
except PasswordExpired:
if self.rotating_password is not None:
warning('[%s] Your password has expired. Switching...' % self.username)
self.change_password(self.rotating_password)
return func(self, *args, **kwargs)
else:
raise
return inner
@check_expired_password
def get_accounts_list(self):
if not self.is_on_page(pages.AccountsList):
self.location('/NSFR?Action=DSP_VGLOBALE')
return self.page.get_list()
def get_account(self, id):
assert isinstance(id, basestring)
if not self.is_on_page(pages.AccountsList):
self.location('/NSFR?Action=DSP_VGLOBALE')
l = self.page.get_list()
for a in l:
if a.id == id:
return a
return None
def get_history(self, account):
if not self.is_on_page(pages.AccountHistory) or self.page.account.id != account.id:
self.location('/SAF_ROP?ch4=%s' % account.link_id)
return self.page.get_operations()
def get_coming_operations(self, account):
if not self.is_on_page(pages.AccountComing) or self.page.account.id != account.id:
self.location('/NS_AVEDT?ch4=%s' % account.link_id)
return self.page.get_operations()
def get_transfer_accounts(self):
if not self.is_on_page(pages.TransferPage):
self.location('/NS_VIRDF')
assert self.is_on_page(pages.TransferPage)
return self.page.get_accounts()
def transfer(self, from_id, to_id, amount, reason=None):
if not self.is_on_page(pages.TransferPage):
self.location('/NS_VIRDF')
accounts = self.page.get_accounts()
self.page.transfer(from_id, to_id, amount, reason)
if not self.is_on_page(pages.TransferCompletePage):
raise TransferError('An error occured during transfer')
transfer = Transfer(self.page.get_id())
transfer.amount = amount
transfer.origin = accounts[from_id].label
transfer.recipient = accounts[to_id].label
transfer.date = datetime.now()
return transfer
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bnporc/captcha.py 0000664 0000000 0000000 00000006665 11666415431 0027154 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import hashlib
import sys
import Image
class TileError(Exception):
def __init__(self, msg, tile = None):
Exception.__init__(self, msg)
self.tile = tile
class Captcha(object):
def __init__(self, file):
self.inim = Image.open(file)
self.nx,self.ny = self.inim.size
self.inmat = self.inim.load()
self.map = {}
self.tiles = [[Tile(x+5*y+1) for y in xrange(5)] for x in xrange(5)]
def __getitem__(self, (x, y)):
return self.inmat[x % self.nx, y % self.ny]
def all_coords(self):
for y in xrange(self.ny):
for x in xrange(self.nx):
yield x, y
def get_codes(self, code):
s = ''
for c in code:
s += '%02d' % self.map[int(c)].id
return s
def build_tiles(self):
y = 1
ty = 0
while y < self.ny:
x = 1
tx = 0
while x < self.nx:
tile = self.tiles[tx][ty]
for j in xrange(26):
l = []
tile.map.append(l)
for i in xrange(26):
if self[x+i,y+j] > 20:
l.append('.')
tile.valid = True
else:
l.append(' ')
if tile.valid:
self.map[tile.get_num()] = tile
x += 27
tx += 1
y += 27
ty += 1
class Tile(object):
hash = {'4a6eff78f6c6f172b75bf9fd7fd36d5d': 0,
'70019df58ec6e96d983507de86529058': 1,
'683a3700dbd1b9019b5ad3ca39c545d3': 2,
'998935d6f4111bd586001468a9c705a7': 3,
'a5cca8bf800fa505cf7ae5039b0cc73c': 4,
'2317a585e19c4f245cdc8acda51e4542': 5,
'956958628d014f6e6bf59d88cd254dc6': 6,
'13c35a4e7bf18e95186311876e66dd95': 7,
'736894876d76899a5cfecc745b095121': 8,
'ff41cd68224bece411c7fc876ab05a1d': 9
}
def __init__(self, _id):
self.id = _id
self.valid = False
self.map = []
def __repr__(self):
return "" % (self.id, self.valid)
def checksum(self):
s = ''
for pxls in self.map:
for pxl in pxls:
s += pxl
return hashlib.md5(s).hexdigest()
def get_num(self):
sum = self.checksum()
try:
return self.hash[sum]
except KeyError:
raise TileError('Tile not found', self)
def display(self):
for pxls in self.map:
for pxl in pxls:
sys.stdout.write(pxl)
sys.stdout.write('\n')
print self.checksum()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bnporc/errors.py 0000664 0000000 0000000 00000001454 11666415431 0027054 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
__all__ = ['PasswordExpired']
class PasswordExpired(Exception):
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bnporc/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026261 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bnporc/pages/__init__.py 0000664 0000000 0000000 00000002365 11666415431 0030400 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .accounts_list import AccountsList
from .account_coming import AccountComing
from .account_history import AccountHistory
from .transfer import TransferPage, TransferConfirmPage, TransferCompletePage
from .login import LoginPage, ConfirmPage, ChangePasswordPage
class AccountPrelevement(AccountsList): pass
__all__ = ['AccountsList', 'AccountComing', 'AccountHistory', 'LoginPage',
'ConfirmPage', 'AccountPrelevement', 'ChangePasswordPage',
'TransferPage', 'TransferConfirmPage', 'TransferCompletePage']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bnporc/pages/account_coming.py 0000664 0000000 0000000 00000005105 11666415431 0031624 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from datetime import date
from weboob.tools.browser import BasePage
from weboob.capabilities.bank import Operation
__all__ = ['AccountComing']
class AccountComing(BasePage):
LABEL_PATTERNS = [('^FACTURECARTEDU(?P
\d{2})(?P\d{2})(?P\d{2})(?P.*)',
u'CB %(yy)s-%(mm)s-%(dd)s: %(text)s'),
('^PRELEVEMENT(?P.*)', 'Order: %(text)s'),
('^ECHEANCEPRET(?P.*)', u'Loan payment n°%(text)s'),
]
def on_loaded(self):
self.operations = []
for tr in self.document.getiterator('tr'):
if tr.attrib.get('class', '') == 'hdoc1' or tr.attrib.get('class', '') == 'hdotc1':
tds = tr.findall('td')
if len(tds) != 3:
continue
d = tds[0].getchildren()[0].attrib.get('name', '')
d = date(int(d[0:4]), int(d[4:6]), int(d[6:8]))
label = u''
label += tds[1].text or u''
label = label.replace(u'\xa0', u'')
for child in tds[1].getchildren():
if child.text: label += child.text
if child.tail: label += child.tail
if tds[1].tail: label += tds[1].tail
label = label.strip()
for pattern, text in self.LABEL_PATTERNS:
m = re.match(pattern, label)
if m:
label = text % m.groupdict()
amount = tds[2].text.replace('.', '').replace(',', '.')
operation = Operation(len(self.operations))
operation.date = d
operation.label = label
operation.amount = float(amount)
self.operations.append(operation)
def get_operations(self):
return self.operations
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bnporc/pages/account_history.py0000664 0000000 0000000 00000005652 11666415431 0032060 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from datetime import date
from weboob.tools.browser import BasePage
from weboob.capabilities.bank import Operation
from weboob.capabilities.base import NotAvailable
__all__ = ['AccountHistory']
class AccountHistory(BasePage):
LABEL_PATTERNS = [(u'^CHEQUEN°(?P.*)', u'CHEQUE', u'N°%(no)s')]
def on_loaded(self):
self.operations = []
for tr in self.document.getiterator('tr'):
if tr.attrib.get('class', '') == 'hdoc1' or tr.attrib.get('class', '') == 'hdotc1':
tds = tr.findall('td')
if len(tds) != 4:
continue
d = date(*reversed([int(x) for x in tds[0].text.split('/')]))
label = u''
label += tds[1].text
label = label.replace(u'\xa0', u'')
for child in tds[1].getchildren():
if child.text: label += child.text
if child.tail: label += child.tail
if tds[1].tail: label += tds[1].tail
label = label.strip()
category = NotAvailable
for pattern, _cat, _lab in self.LABEL_PATTERNS:
m = re.match(pattern, label)
if m:
category = _cat % m.groupdict()
label = _lab % m.groupdict()
break
else:
if ' ' in label:
category, useless, label = [part.strip() for part in label.partition(' ')]
amount = tds[2].text.replace('.', '').replace(',', '.')
# if we don't have exactly one '.', this is not a floatm try the next
operation = Operation(len(self.operations))
if amount.count('.') != 1:
amount = tds[3].text.replace('.', '').replace(',', '.')
operation.amount = float(amount)
else:
operation.amount = - float(amount)
operation.date = d
operation.label = label
operation.category = category
self.operations.append(operation)
def get_operations(self):
return self.operations
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bnporc/pages/accounts_list.py 0000664 0000000 0000000 00000006027 11666415431 0031512 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from weboob.capabilities.bank import Account
from weboob.capabilities.base import NotAvailable
from weboob.tools.browser import BasePage
from ..errors import PasswordExpired
__all__ = ['AccountsList']
class AccountsList(BasePage):
LINKID_REGEXP = re.compile(".*ch4=(\w+).*")
def on_loaded(self):
pass
def get_list(self):
l = []
for tr in self.document.getiterator('tr'):
if tr.attrib.get('class', '') == 'comptes':
account = Account()
for td in tr.getiterator('td'):
if td.attrib.get('headers', '').startswith('Numero_'):
id = td.text
account.id = ''.join(id.split(' ')).strip()
elif td.attrib.get('headers', '').startswith('Libelle_'):
a = td.findall('a')
label = unicode(a[0].text)
account.label = label.strip()
m = self.LINKID_REGEXP.match(a[0].attrib.get('href', ''))
if m:
account.link_id = m.group(1)
elif td.attrib.get('headers', '').startswith('Solde'):
a = td.findall('a')
balance = a[0].text
balance = balance.replace('.','').replace(',','.')
account.balance = float(balance)
elif td.attrib.get('headers', '').startswith('Avenir'):
a = td.findall('a')
# Some accounts don't have a "coming"
if len(a):
coming = a[0].text
coming = coming.replace('.','').replace(',','.')
account.coming = float(coming)
else:
account.coming = NotAvailable
l.append(account)
if len(l) == 0:
# oops, no accounts? check if we have not exhausted the allowed use
# of this password
for div in self.document.getroot().cssselect('div.Style_texte_gras'):
if div.text.strip() == 'Vous avez atteint la date de fin de vie de votre code secret.':
raise PasswordExpired(div.text.strip())
return l
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bnporc/pages/login.py 0000664 0000000 0000000 00000005063 11666415431 0027747 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.mech import ClientForm
import urllib
from logging import error
from weboob.tools.browser import BasePage, BrowserUnavailable
from weboob.backends.bnporc.captcha import Captcha, TileError
__all__ = ['LoginPage', 'ConfirmPage', 'ChangePasswordPage']
class LoginPage(BasePage):
def on_loaded(self):
for td in self.document.getroot().cssselect('td.LibelleErreur'):
if td.text is None:
continue
msg = td.text.strip()
if 'indisponible' in msg:
raise BrowserUnavailable(msg)
def login(self, login, password):
img = Captcha(self.browser.openurl('/NSImgGrille'))
try:
img.build_tiles()
except TileError, err:
error("Error: %s" % err)
if err.tile:
err.tile.display()
self.browser.select_form('logincanalnet')
# HACK because of fucking malformed HTML, the field isn't recognized by mechanize.
self.browser.controls.append(ClientForm.TextControl('text', 'ch1', {'value': ''}))
self.browser.set_all_readonly(False)
self.browser['ch1'] = login
self.browser['ch5'] = img.get_codes(password)
self.browser.submit()
class ConfirmPage(BasePage):
pass
class ChangePasswordPage(BasePage):
def change_password(self, current, new):
img = Captcha(self.browser.openurl('/NSImgGrille'))
try:
img.build_tiles()
except TileError, err:
error('Error: %s' % err)
if err.tile:
err.tile.display()
code_current = img.get_codes(current)
code_new = img.get_codes(new)
data = {'ch1': code_current,
'ch2': code_new,
'ch3': code_new
}
self.browser.location('/SAF_CHM_VALID', urllib.urlencode(data))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bnporc/pages/transfer.py 0000664 0000000 0000000 00000007026 11666415431 0030464 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from weboob.tools.browser import BasePage
from weboob.tools.ordereddict import OrderedDict
from weboob.capabilities.bank import TransferError
__all__ = ['TransferPage', 'TransferConfirmPage', 'TransferCompletePage']
class Account(object):
def __init__(self, id, label, send_checkbox, receive_checkbox):
self.id = id
self.label = label
self.send_checkbox = send_checkbox
self.receive_checkbox = receive_checkbox
class TransferPage(BasePage):
def get_accounts(self):
accounts = OrderedDict()
for table in self.document.getiterator('table'):
if table.attrib.get('cellspacing') == '2':
for tr in table.cssselect('tr.hdoc1, tr.hdotc1'):
tds = tr.findall('td')
id = tds[1].text.replace(u'\xa0', u'')
label = tds[0].text
if label is None and tds[0].find('nobr') is not None:
label = tds[0].find('nobr').text
send_checkbox = tds[4].find('input').attrib['value'] if tds[4].find('input') is not None else None
receive_checkbox = tds[5].find('input').attrib['value'] if tds[5].find('input') is not None else None
account = Account(id, label, send_checkbox, receive_checkbox)
accounts[id] = account
return accounts
def transfer(self, from_id, to_id, amount, reason):
accounts = self.get_accounts()
try:
sender = accounts[from_id]
except KeyError:
raise TransferError('Account %s not found' % from_id)
try:
recipient = accounts[to_id]
except KeyError:
raise TransferError('Recipient %s not found' % to_id)
if sender.send_checkbox is None:
raise TransferError('Unable to make a transfer from %s' % sender.label)
if recipient.receive_checkbox is None:
raise TransferError('Unable to make a transfer to %s' % recipient.label)
self.browser.select_form(nr=0)
self.browser['C1'] = [sender.send_checkbox]
self.browser['C2'] = [recipient.receive_checkbox]
self.browser['T6'] = str(amount).replace('.', ',')
if reason:
self.browser['T5'] = reason.encode('utf-8')
self.browser.submit()
class TransferConfirmPage(BasePage):
def on_loaded(self):
for td in self.document.getroot().cssselect('td#size2'):
raise TransferError(td.text.strip())
for a in self.document.getiterator('a'):
m = re.match('/NSFR\?Action=VIRDA&stp=(\d+)', a.attrib['href'])
if m:
self.browser.location('/NS_VIRDA?stp=%s' % m.group(1))
return
class TransferCompletePage(BasePage):
def get_id(self):
return self.group_dict['id']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bnporc/test.py 0000664 0000000 0000000 00000002031 11666415431 0026507 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class BNPorcTest(BackendTest):
BACKEND = 'bnporc'
def test_bnporc(self):
l = list(self.backend.iter_accounts())
if len(l) > 0:
a = l[0]
list(self.backend.iter_operations(a))
list(self.backend.iter_history(a))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bouygues/ 0000775 0000000 0000000 00000000000 11666415431 0025541 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bouygues/__init__.py 0000664 0000000 0000000 00000000104 11666415431 0027645 0 ustar 00root root 0000000 0000000 from .backend import BouyguesBackend
__all__ = ['BouyguesBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bouygues/backend.py 0000664 0000000 0000000 00000003531 11666415431 0027504 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.messages import CantSendMessage, ICapMessages, ICapMessagesPost
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import ValueBackendPassword, Value
from .browser import BouyguesBrowser
__all__ = ['BouyguesBackend']
class BouyguesBackend(BaseBackend, ICapMessages, ICapMessagesPost):
NAME = 'bouygues'
MAINTAINER = 'Christophe Benz'
EMAIL = 'christophe.benz@gmail.com'
VERSION = '0.9.1'
DESCRIPTION = 'Bouygues french mobile phone provider'
LICENSE = 'AGPLv3+'
CONFIG = BackendConfig(Value('login', label='Login'),
ValueBackendPassword('password', label='Password'))
BROWSER = BouyguesBrowser
ACCOUNT_REGISTER_PROPERTIES = None
def create_default_browser(self):
return self.create_browser(self.config['login'].get(), self.config['password'].get())
def post_message(self, message):
if not message.content.strip():
raise CantSendMessage(u'Message content is empty.')
with self.browser:
self.browser.post_message(message)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bouygues/browser.py 0000664 0000000 0000000 00000005044 11666415431 0027601 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .pages.compose import ComposeFrame, ComposePage, ConfirmPage, SentPage
from .pages.login import LoginPage, LoginSASPage
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
__all__ = ['BouyguesBrowser']
class BouyguesBrowser(BaseBrowser):
DOMAIN = 'www.bouyguestelecom.fr'
PAGES = {
'http://www.espaceclient.bouyguestelecom.fr/ECF/jsf/client/envoiSMS/viewEnvoiSMS.jsf': ComposePage,
'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/sendSMS.phtml': ComposeFrame,
'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/confirmSendSMS.phtml': ConfirmPage,
'https://www.espaceclient.bouyguestelecom.fr/ECF/jsf/submitLogin.jsf': LoginPage,
'https://www.espaceclient.bouyguestelecom.fr/ECF/SasUnifie': LoginSASPage,
'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/resultSendSMS.phtml': SentPage,
}
def home(self):
self.location('http://www.espaceclient.bouyguestelecom.fr/ECF/jsf/client/envoiSMS/viewEnvoiSMS.jsf')
def is_logged(self):
return 'code' not in [form.name for form in self.forms()]
def login(self):
self.location('https://www.espaceclient.bouyguestelecom.fr/ECF/jsf/submitLogin.jsf', no_login=True)
self.page.login(self.username, self.password)
assert self.is_on_page(LoginSASPage)
self.page.login()
if not self.is_logged():
raise BrowserIncorrectPassword()
def post_message(self, message):
if not self.is_on_page(ComposeFrame):
self.home()
self.location('http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/sendSMS.phtml')
self.page.post_message(message)
assert self.is_on_page(ConfirmPage)
self.page.confirm()
assert self.is_on_page(SentPage)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bouygues/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026640 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bouygues/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0030737 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bouygues/pages/compose.py 0000664 0000000 0000000 00000003156 11666415431 0030664 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from weboob.capabilities.messages import CantSendMessage
from weboob.tools.browser import BasePage
__all__ = ['ComposeFrame', 'ComposePage', 'ConfirmPage', 'SentPage']
class ComposeFrame(BasePage):
phone_regex = re.compile('^(\+33|0033|0)(6|7)(\d{8})$')
def post_message(self, message):
receiver = message.thread.id
if self.phone_regex.match(receiver) is None:
raise CantSendMessage(u'Invalid receiver: %s' % receiver)
self.browser.select_form(nr=0)
self.browser['fieldMsisdn'] = receiver
self.browser['fieldMessage'] = message.content.encode('utf-8')
self.browser.submit()
class ComposePage(BasePage):
pass
class ConfirmPage(BasePage):
def confirm(self):
self.browser.location('http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/resultSendSMS.phtml')
class SentPage(BasePage):
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bouygues/pages/login.py 0000664 0000000 0000000 00000002243 11666415431 0030323 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage
__all__ = ['LoginPage', 'LoginSASPage']
class LoginPage(BasePage):
def login(self, login, password):
self.browser.select_form(name='code')
self.browser['j_username'] = login
self.browser['j_password'] = password
self.browser.submit()
class LoginSASPage(BasePage):
def login(self):
self.browser.select_form(name='redirect')
self.browser.submit()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bouygues/test.py 0000664 0000000 0000000 00000001562 11666415431 0027076 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class BouyguesTest(BackendTest):
BACKEND = 'bouygues'
def test_bouygues(self):
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bp/ 0000775 0000000 0000000 00000000000 11666415431 0024300 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bp/__init__.py 0000664 0000000 0000000 00000001463 11666415431 0026415 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
#
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import BPBackend
__all__ = ['BPBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bp/backend.py 0000664 0000000 0000000 00000004327 11666415431 0026247 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.bank import ICapBank, AccountNotFound
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import ValueBackendPassword
from .browser import BPBrowser
__all__ = ['BPBackend']
class BPBackend(BaseBackend, ICapBank):
NAME = 'bp'
MAINTAINER = 'Nicolas Duhamel'
EMAIL = 'nicolas@jombi.fr'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = u'La banque postale, French bank'
CONFIG = BackendConfig(ValueBackendPassword('login', label='Account ID', masked=False),
ValueBackendPassword('password', label='Password'))
BROWSER = BPBrowser
def create_default_browser(self):
return self.create_browser(self.config['login'].get(), self.config['password'].get())
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
def iter_history(self, account):
for history in self.browser.get_history(account):
yield history
def transfer(self, id_from, id_to, amount, reason=None):
from_account = self.get_account(id_from)
to_account = self.get_account(id_to)
#TODO: retourner le numero du virement
#TODO: support the 'reason' parameter
return self.browser.make_transfer(from_account, to_account, amount)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bp/browser.py 0000664 0000000 0000000 00000012057 11666415431 0026342 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import datetime
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword, BrowserBanned
from .pages import LoginPage, Initident, CheckPassword, repositionnerCheminCourant, BadLoginPage, AccountDesactivate, \
AccountList, AccountHistory, \
TransferChooseAccounts, CompleteTransfer, TransferConfirm, TransferSummary
from weboob.capabilities.bank import Transfer
__all__ = ['BPBrowser']
class BPBrowser(BaseBrowser):
DOMAIN = 'voscomptesenligne.labanquepostale.fr'
PROTOCOL = 'https'
ENCODING = None # refer to the HTML encoding
PAGES = {r'.*wsost/OstBrokerWeb/loginform.*' : LoginPage,
r'.*authentification/repositionnerCheminCourant-identif.ea' : repositionnerCheminCourant,
r'.*authentification/initialiser-identif.ea' : Initident,
r'.*authentification/verifierMotDePasse-identif.ea' : CheckPassword,
r'.*synthese_assurancesEtComptes/afficheSynthese-synthese\.ea' : AccountList,
r'.*synthese_assurancesEtComptes/rechercheContratAssurance-synthese.ea' : AccountList,
r'.*CCP/releves_ccp/releveCPP-releve_ccp\.ea' : AccountHistory,
r'.*CNE/releveCNE/releveCNE-releve_cne\.ea' : AccountHistory,
r'.*/virementSafran_aiguillage/init-saisieComptes\.ea' : TransferChooseAccounts,
r'.*/virementSafran_aiguillage/formAiguillage-saisieComptes\.ea' : CompleteTransfer,
r'.*/virementSafran_national/validerVirementNational-virementNational.ea' : TransferConfirm,
r'.*/virementSafran_national/confirmerVirementNational-virementNational.ea' : TransferSummary,
r'.*ost/messages\.CVS\.html\?param=0x132120c8.*' : BadLoginPage,
r'.*ost/messages\.CVS\.html\?param=0x132120cb.*' : AccountDesactivate,
}
def __init__(self, *args, **kwargs):
kwargs['parser'] = ('lxml',)
BaseBrowser.__init__(self, *args, **kwargs)
def home(self):
self.location('https://voscomptesenligne.labanquepostale.fr/wsost/OstBrokerWeb/loginform?TAM_OP=login&'
'ERROR_CODE=0x00000000&URL=%2Fvoscomptes%2FcanalXHTML%2Fidentif.ea%3Forigin%3Dparticuliers')
def is_logged(self):
return not self.is_on_page(LoginPage)
def login(self):
if not self.is_on_page(LoginPage):
self.location('https://voscomptesenligne.labanquepostale.fr/wsost/OstBrokerWeb/loginform?TAM_OP=login&'
'ERROR_CODE=0x00000000&URL=%2Fvoscomptes%2FcanalXHTML%2Fidentif.ea%3Forigin%3Dparticuliers',
no_login=True)
self.page.login(self.username, self.password)
if self.is_on_page(BadLoginPage):
raise BrowserIncorrectPassword()
if self.is_on_page(AccountDesactivate):
raise BrowserBanned()
def get_accounts_list(self):
self.location("https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/comptesCommun/synthese_assurancesEtComptes/rechercheContratAssurance-synthese.ea")
return self.page.get_accounts_list()
def get_account(self, id):
if not self.is_on_page(AccountList):
self.location("https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/comptesCommun/synthese_assurancesEtComptes/rechercheContratAssurance-synthese.ea")
return self.page.get_account(id)
def get_history(self, Account):
self.location(Account.link_id)
return self.page.get_history()
def make_transfer(self, from_account, to_account, amount):
self.location('https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/virement/virementSafran_aiguillage/init-saisieComptes.ea')
self.page.set_accouts(from_account, to_account)
#TODO: Check
self.page.complete_transfer(amount)
self.page.confirm()
id_transfer = self.page.get_transfer_id()
transfer = Transfer(id_transfer)
transfer.amount = amount
transfer.origin = from_account.label
transfer.recipient = to_account.label
transfer.date = datetime.now()
return transfer
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bp/pages/ 0000775 0000000 0000000 00000000000 11666415431 0025377 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bp/pages/__init__.py 0000664 0000000 0000000 00000002402 11666415431 0027506 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .login import LoginPage, Initident, CheckPassword,repositionnerCheminCourant, BadLoginPage, AccountDesactivate
from .accountlist import AccountList
from .accounthistory import AccountHistory
from .transfer import TransferChooseAccounts, CompleteTransfer, TransferConfirm, TransferSummary
__all__ = ['LoginPage','Initident', 'CheckPassword', 'repositionnerCheminCourant', "AccountList", 'AccountHistory', 'BadLoginPage',
'AccountDesactivate', 'TransferChooseAccounts', 'CompleteTransfer', 'TransferConfirm', 'TransferSummary']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bp/pages/accounthistory.py 0000664 0000000 0000000 00000003716 11666415431 0031036 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from weboob.capabilities.bank import Operation
from weboob.tools.browser import BasePage
from weboob.tools.misc import remove_html_tags
__all__ = ['AccountHistory']
class AccountHistory(BasePage):
def get_history(self):
mvt_table = self.document.xpath("//table[@id='mouvements']", smart_strings=False)[0]
mvt_ligne = mvt_table.xpath("./tbody/tr")
operations = []
for mvt in mvt_ligne:
operation = Operation(len(operations))
operation.date = mvt.xpath("./td/span")[0].text
tmp = mvt.xpath("./td/span")[1]
operation.label = remove_html_tags(self.parser.tostring(tmp)).strip()
r = re.compile(r'\d+')
tmp = mvt.xpath("./td/span/strong")
if not tmp:
tmp = mvt.xpath("./td/span")
amount = None
for t in tmp:
if r.search(t.text):
amount = t.text
amount = ''.join( amount.replace('.', '').replace(',', '.').split() )
if amount[0] == "-":
operation.amount = -float(amount[1:])
else:
operation.amount = float(amount)
operations.append(operation)
return operations
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bp/pages/accountlist.py 0000664 0000000 0000000 00000004363 11666415431 0030307 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.bank import Account, AccountNotFound
from weboob.tools.browser import BasePage
__all__ = ['AccountList']
class AccountList(BasePage):
def on_loaded(self):
self.account_list = []
self.parse_table('comptes')
self.parse_table('comptesEpargne')
self.parse_table('comptesTitres')
self.parse_table('comptesVie')
self.parse_table('comptesRetraireEuros')
def get_accounts_list(self):
return self.account_list
def parse_table(self, what):
tables = self.document.xpath("//table[@id='%s']" % what, smart_strings=False)
if len(tables) < 1:
return
lines = tables[0].xpath(".//tbody/tr")
for line in lines:
account = Account()
tmp = line.xpath("./td//a")[0]
account.label = tmp.text
account.link_id = tmp.get("href")
tmp = line.xpath("./td//strong")
if len(tmp) != 2:
tmp_id = line.xpath("./td//span")[1].text
tmp_balance = tmp[0].text
else:
tmp_id = tmp[0].text
tmp_balance = tmp[1].text
account.id = tmp_id
account.balance = float(''.join(tmp_balance.replace('.','').replace(',','.').split()))
self.account_list.append(account)
def get_account(self, id):
for account in self.account_list:
if account.id == id:
return account
raise AccountNotFound('Unable to find account: %s' % id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bp/pages/login.py 0000664 0000000 0000000 00000005533 11666415431 0027067 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import hashlib
from weboob.tools.browser import BasePage
__all__ = ['LoginPage', 'BadLoginPage', 'AccountDesactivate', 'Initident', 'CheckPassword', 'repositionnerCheminCourant']
def md5(file):
f = open(file,'rb')
md5 = hashlib.md5()
md5.update(f.read())
return md5.hexdigest()
class LoginPage(BasePage):
def on_loaded(self):
pass
def login(self, login, pwd):
LOCAL_HASH = ['a02574d7bf67677d2a86b7bfc5e864fe', 'eb85e1cc45dd6bdb3cab65c002d7ac8a',
'596e6fbd54d5b111fe5df8a4948e80a4', '9cdc989a4310554e7f5484d0d27a86ce',
'0183943de6c0e331f3b9fc49c704ac6d', '291b9987225193ab1347301b241e2187',
'163279f1a46082408613d12394e4042a', 'b0a9c740c4cada01eb691b4acda4daea',
'3c4307ee92a1f3b571a3c542eafcb330', 'dbccecfa2206bfdb4ca891476404cc68']
process = lambda i: md5(self.browser.retrieve(('https://voscomptesenligne.labanquepostale.fr/wsost/OstBrokerWeb/loginform?imgid=%d&0.25122230781963073' % i))[0])
keypad = [process(i) for i in range(10)]
correspondance = [keypad.index(i) for i in LOCAL_HASH]
newpassword = ''.join(str(correspondance[int(c)]) for c in pwd)
self.browser.select_form(name='formAccesCompte')
self.browser.set_all_readonly(False)
self.browser['password'] = newpassword
self.browser['username'] = login
self.browser.submit()
class repositionnerCheminCourant(BasePage):
def on_loaded(self):
self.browser.open("https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/securite/authentification/initialiser-identif.ea")
class Initident(BasePage):
def on_loaded(self):
self.browser.open("https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/securite/authentification/verifierMotDePasse-identif.ea")
class CheckPassword(BasePage):
def on_loaded(self):
self.browser.open("https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/comptesCommun/synthese_assurancesEtComptes/init-synthese.ea")
class BadLoginPage(BasePage):
pass
class AccountDesactivate(BasePage):
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/bp/pages/transfer.py 0000664 0000000 0000000 00000005216 11666415431 0027601 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from weboob.capabilities.bank import TransferError
from weboob.tools.browser import BasePage
from weboob.tools.misc import to_unicode
__all__ = ['TransferChooseAccounts', 'CompleteTransfer', 'TransferConfirm', 'TransferSummary']
class TransferChooseAccounts(BasePage):
def set_accouts(self, from_account, to_account):
self.browser.select_form(name="AiguillageForm")
self.browser["idxCompteEmetteur"] = [from_account.id]
self.browser["idxCompteReceveur"] = [to_account.id]
self.browser.submit()
class CompleteTransfer(BasePage):
def complete_transfer(self, amount):
self.browser.select_form(name="virement_unitaire_saisie_saisie_virement_sepa")
self.browser["montant"] = str(amount)
self.browser.submit()
class TransferConfirm(BasePage):
def confirm(self):
self.browser.location('https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/virement/virementSafran_national/confirmerVirementNational-virementNational.ea')
class TransferSummary(BasePage):
def get_transfer_id(self):
p = self.document.xpath("//div[@id='main']/div/p")[0]
#HACK for deal with bad encoding ...
try:
text = p.text
except UnicodeDecodeError, error:
text = error.object.strip()
match = re.search("Votre virement N.+ ([0-9]+) ", text)
if match:
id_transfer = match.groups()[0]
return id_transfer
if text.startswith(u"Votre virement n'a pas pu"):
if p.find('br') is not None:
errmsg = to_unicode(p.find('br').tail).strip()
raise TransferError('Unable to process transfer: %s' % errmsg)
else:
self.browser.logger.warning('Unable to find the error reason')
self.browser.logger.error('Unable to parse the text result: %r' % text)
raise TransferError('Unable to process transfer: %r' % text)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canalplus/ 0000775 0000000 0000000 00000000000 11666415431 0025661 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canalplus/__init__.py 0000664 0000000 0000000 00000001447 11666415431 0030000 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import CanalplusBackend
__all__ = ['CanalplusBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canalplus/backend.py 0000664 0000000 0000000 00000004746 11666415431 0027635 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.video import ICapVideo
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import Value
from .browser import CanalplusBrowser
from .pages import CanalplusVideo
from weboob.capabilities.collection import ICapCollection
__all__ = ['CanalplusBackend']
class CanalplusBackend(BaseBackend, ICapVideo, ICapCollection):
NAME = 'canalplus'
MAINTAINER = 'Nicolas Duhamel'
EMAIL = 'nicolas@jombi.fr'
VERSION = '0.9.1'
DESCRIPTION = 'Canal plus french TV'
LICENSE = 'AGPLv3+'
CONFIG = BackendConfig(Value('quality', label='Quality of videos', choices=['hd', 'sd'], default='hd'))
BROWSER = CanalplusBrowser
def create_default_browser(self):
return self.create_browser(quality=self.config['quality'].get())
def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
with self.browser:
return self.browser.iter_search_results(pattern)
def get_video(self, _id):
with self.browser:
return self.browser.get_video(_id)
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
with self.browser:
video = self.browser.get_video(CanalplusVideo.id2url(video.id), video)
if 'thumbnail' in fields and video.thumbnail:
with self.browser:
video.thumbnail.data = self.browser.readurl(video.thumbnail.url)
return video
OBJECTS = {CanalplusVideo: fill_video}
def iter_resources(self, splited_path):
with self.browser:
return self.browser.iter_resources(splited_path)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canalplus/browser.py 0000664 0000000 0000000 00000006234 11666415431 0027723 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import urllib
import lxml.etree
from weboob.tools.browser import BaseBrowser
from weboob.tools.browser.decorators import id2url
from .pages import InitPage, CanalplusVideo, VideoPage
from weboob.capabilities.collection import Collection, CollectionNotFound
__all__ = ['CanalplusBrowser']
class XMLParser(object):
def parse(self, data, encoding=None):
if encoding is None:
parser = None
else:
parser = lxml.etree.XMLParser(encoding=encoding, strip_cdata=False)
return lxml.etree.XML(data.get_data(), parser)
class CanalplusBrowser(BaseBrowser):
DOMAIN = u'service.canal-plus.com'
ENCODING = 'utf-8'
PAGES = {
r'http://service.canal-plus.com/video/rest/initPlayer/cplus/': InitPage,
r'http://service.canal-plus.com/video/rest/search/cplus/.*': VideoPage,
r'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/(?P.+)': VideoPage,
r'http://service.canal-plus.com/video/rest/getMEAs/cplus/.*': VideoPage,
}
#We need lxml.etree.XMLParser for read CDATA
PARSER = XMLParser()
FORMATS = {
'sd': 'BAS_DEBIT',
'hd': 'HD',
}
def __init__(self, quality, *args, **kwargs):
BaseBrowser.__init__(self, parser= self.PARSER, *args, **kwargs)
if quality in self.FORMATS:
self.quality = self.FORMATS[quality]
else:
self.quality = 'HD'
def home(self):
self.location('http://service.canal-plus.com/video/rest/initPlayer/cplus/')
def iter_search_results(self, pattern):
self.location('http://service.canal-plus.com/video/rest/search/cplus/' + urllib.quote_plus(pattern.encode('utf-8')))
return self.page.iter_results()
@id2url(CanalplusVideo.id2url)
def get_video(self, url, video=None):
self.location(url)
return self.page.get_video(video, self.quality)
def iter_resources(self, splited_path):
self.home()
collections = self.page.collections
def walk_res(path, collections):
if len(path) == 0 or not isinstance(collections, (list, Collection)):
return collections
i = path[0]
if i not in [collection.title for collection in collections]:
raise CollectionNotFound()
return walk_res(path[1:], [collection.children for collection in collections if collection.title == i][0])
return walk_res(splited_path, collections)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canalplus/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026760 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canalplus/pages/__init__.py 0000664 0000000 0000000 00000001573 11666415431 0031077 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .initpage import InitPage
from .video import CanalplusVideo
from .videopage import VideoPage
__all__ = ['InitPage', 'VideoPage', 'CanalplusVideo']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canalplus/pages/initpage.py 0000664 0000000 0000000 00000003306 11666415431 0031134 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage
from weboob.capabilities.collection import Collection
__all__ = ['InitPage']
class InitPage(BasePage):
def on_loaded(self):
self.collections = []
def do(id):
self.browser.location("http://service.canal-plus.com/video/rest/getMEAs/cplus/" + id)
return self.browser.page.iter_channel()
### Parse liste des channels
for elem in self.document[2].getchildren():
coll = Collection()
for e in elem.getchildren():
if e.tag == "NOM":
coll.title = e.text.strip().encode('utf-8')
elif e.tag == "SELECTIONS":
for select in e:
sub = Collection(title=select[1].text.strip().encode('utf-8'))
sub.id = select[0].text
sub.children = do
coll.appendchild(sub)
self.collections.append(coll)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canalplus/pages/video.py 0000664 0000000 0000000 00000001756 11666415431 0030451 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.video import BaseVideo
__all__ = ['CanalplusVideo']
class CanalplusVideo(BaseVideo):
swf_player = False
@classmethod
def id2url(cls, _id):
return 'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/%s' % _id
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canalplus/pages/videopage.py 0000664 0000000 0000000 00000006124 11666415431 0031300 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import datetime
from weboob.capabilities.base import NotAvailable
from weboob.tools.capabilities.thumbnail import Thumbnail
from weboob.tools.browser import BasePage
from .video import CanalplusVideo
__all__ = ['VideoPage']
class VideoPage(BasePage):
def parse_video(self, el, video=None, quality=None):
_id = el.find('ID').text
if _id == '-1':
# means the video is not found
return None
if not video:
video = CanalplusVideo(_id)
infos = el.find('INFOS')
video.title = u''
for part in infos.find('TITRAGE'):
if len(part.text.strip()) == 0:
continue
if len(video.title) > 0:
video.title += u' — '
video.title += part.text.strip()
video.description = infos.find('DESCRIPTION').text
media = el.find('MEDIA')
url = media.find('IMAGES').find('PETIT').text
if url:
video.thumbnail = Thumbnail(url)
else:
video.thumbnail = NotAvailable
lastest_format = None
for format in media.find('VIDEOS'):
if format.text is None:
continue
if format.tag == quality:
video.url = format.text
break
lastest_format = format
if not video.url and lastest_format is not None:
video.url = lastest_format.text
day, month, year = map(int, infos.find('PUBLICATION').find('DATE').text.split('/'))
hour, minute, second = map(int, infos.find('PUBLICATION').find('HEURE').text.split(':'))
video.date = datetime(year, month, day, hour, minute, second)
return video
def iter_results(self):
for vid in self.document.getchildren():
yield self.parse_video(vid)
def iter_channel(self):
for vid in self.document.getchildren():
yield self.parse_video_channel(vid)
def parse_video_channel(self,el):
_id = el[0].text
video = CanalplusVideo(_id)
video.title = el[2][3][0].text
video.date = datetime.now()
return video
def get_video(self, video, quality):
_id = self.group_dict['id']
for vid in self.document.getchildren():
if not _id in vid.find('ID').text:
continue
return self.parse_video(vid, video, quality)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canalplus/test.py 0000664 0000000 0000000 00000002166 11666415431 0027217 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class CanalPlusTest(BackendTest):
BACKEND = 'canalplus'
def test_canalplus(self):
l = list(self.backend.iter_search_results('guignol'))
self.assertTrue(len(l) > 0)
v = l[0]
self.backend.fillobj(v, ('url',))
self.assertTrue(v.url and v.url.startswith('rtmp://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canaltp/ 0000775 0000000 0000000 00000000000 11666415431 0025321 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canaltp/__init__.py 0000664 0000000 0000000 00000001442 11666415431 0027433 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import CanalTPBackend
__all__ = ['CanalTPBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canaltp/backend.py 0000664 0000000 0000000 00000003325 11666415431 0027265 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.travel import ICapTravel, Station, Departure
from weboob.tools.backend import BaseBackend
from .browser import CanalTP
__all__ = ['CanalTPBackend']
class CanalTPBackend(BaseBackend, ICapTravel):
NAME = 'canaltp'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = "French trains"
BROWSER = CanalTP
def iter_station_search(self, pattern):
for _id, name in self.browser.iter_station_search(pattern):
yield Station(_id, name)
def iter_station_departures(self, station_id, arrival_id=None):
for i, d in enumerate(self.browser.iter_station_departures(station_id, arrival_id)):
departure = Departure(i, d['type'], d['time'])
departure.departure_station = d['departure']
departure.arrival_station = d['arrival']
departure.late = d['late']
departure.information = d['late_reason']
yield departure
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canaltp/browser.py 0000664 0000000 0000000 00000005425 11666415431 0027364 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import datetime, date, time
from weboob.tools.browser import BaseBrowser
from weboob.tools.misc import to_unicode
from weboob.tools.browser import BrokenPageError
__all__ = ['CanalTP']
class CanalTP(BaseBrowser):
DOMAIN = 'widget.canaltp.fr'
def __init__(self, **kwargs):
BaseBrowser.__init__(self, '', **kwargs)
def iter_station_search(self, pattern):
url = u'http://widget.canaltp.fr/Prochains_departs_15122009/dev/gare.php?txtrech=%s' % unicode(pattern)
result = self.openurl(url.encode('utf-8')).read()
for station in result.split('&'):
try:
_id, name = station.split('=')
except ValueError:
continue
else:
yield _id, to_unicode(name)
def iter_station_departures(self, station_id, arrival_id=None):
url = u'http://widget.canaltp.fr/Prochains_departs_15122009/dev/index.php?gare=%s' % unicode(station_id)
result = self.openurl(url.encode('utf-8')).read()
result = result
departure = ''
for line in result.split('&'):
if not '=' in line:
raise BrokenPageError('Unable to parse result: %s' % line)
key, value = line.split('=', 1)
if key == 'nomgare':
departure = value
elif key.startswith('ligne'):
_type, unknown, _time, arrival, served, late, late_reason = value.split(';', 6)
yield {'type': to_unicode(_type),
'time': datetime.combine(date.today(), time(*[int(x) for x in _time.split(':')])),
'departure': to_unicode(departure),
'arrival': to_unicode(arrival).strip(),
'late': late and time(0, int(late.split()[0])) or time(),
'late_reason': to_unicode(late_reason).replace('\n', '').strip()}
def home(self):
pass
def login(self):
pass
def is_logged(self):
""" Do not need to be logged """
return True
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/canaltp/test.py 0000664 0000000 0000000 00000002024 11666415431 0026650 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class CanalTPTest(BackendTest):
BACKEND = 'canaltp'
def test_canaltp(self):
stations = list(self.backend.iter_station_search('defense'))
self.assertTrue(len(stations) > 0)
list(self.backend.iter_station_departures(stations[0].id))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/cragr/ 0000775 0000000 0000000 00000000000 11666415431 0024775 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/cragr/__init__.py 0000664 0000000 0000000 00000001436 11666415431 0027112 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import CragrBackend
__all__ = ['CragrBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/cragr/backend.py 0000664 0000000 0000000 00000010220 11666415431 0026731 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.bank import ICapBank, AccountNotFound
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.ordereddict import OrderedDict
from weboob.tools.value import ValueBackendPassword, Value
from .browser import Cragr
__all__ = ['CragrBackend']
class CragrBackend(BaseBackend, ICapBank):
NAME = 'cragr'
MAINTAINER = 'Xavier Guerrin'
EMAIL = 'xavier@tuxfamily.org'
VERSION = '0.9.1'
DESCRIPTION = 'Credit Agricole french bank\'s website'
LICENSE = 'AGPLv3+'
website_choices = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted({
'm.ca-alpesprovence.fr': u'Alpes Provence',
'm.ca-anjou-maine.fr': u'Anjou Maine',
'm.ca-atlantique-vendee.fr': u'Atlantique Vendée',
'm.ca-aquitaine.fr': u'Aquitaine',
'm.ca-briepicardie.fr': u'Brie Picardie',
'm.ca-centrest.fr': u'Centre Est',
'm.ca-centrefrance.fr': u'Centre France',
'm.ca-centreloire.fr': u'Centre Loire',
'm.ca-centreouest.fr': u'Centre Ouest',
'm.ca-cb.fr': u'Champagne Bourgogne',
'm.ca-charente-perigord.fr': u'Charente Périgord',
'm.ca-cmds.fr': u'Charente-Maritime Deux-Sèvres',
'm.ca-corse.fr': u'Corse',
'm.ca-cotesdarmor.fr': u'Côtes d\'Armor',
'm.ca-des-savoie.fr': u'Des Savoie',
'm.ca-finistere.fr': u'Finistere',
'm.ca-paris.fr': u'Ile-de-France',
'm.ca-illeetvilaine.fr': u'Ille-et-Vilaine',
'm.ca-languedoc.fr': u'Languedoc',
'm.ca-loirehauteloire.fr': u'Loire Haute Loire',
'm.ca-lorraine.fr': u'Lorraine',
'm.ca-martinique.fr': u'Martinique Guyane',
'm.ca-morbihan.fr': u'Morbihan',
'm.ca-norddefrance.fr': u'Nord de France',
'm.ca-nord-est.fr': u'Nord Est',
'm.ca-nmp.fr': u'Nord Midi-Pyrénées',
'm.ca-normandie.fr': u'Normandie',
'm.ca-normandie-seine.fr': u'Normandie Seine',
'm.ca-pca.fr': u'Provence Côte d\'Azur',
'm.lefil.com': u'Pyrénées Gascogne',
'm.ca-reunion.fr': u'Réunion',
'm.ca-sudrhonealpes.fr': u'Sud Rhône Alpes',
'm.ca-sudmed.fr': u'Sud Méditerranée',
'm.ca-toulouse31.fr': u'Toulouse 31', # m.ca-toulousain.fr redirects here
'm.ca-tourainepoitou.fr': u'Tourraine Poitou',
}.iteritems())])
CONFIG = BackendConfig(Value('website', label='Website to use', choices=website_choices),
ValueBackendPassword('login', label='Account ID', masked=False),
ValueBackendPassword('password', label='Password'))
BROWSER = Cragr
def create_default_browser(self):
return self.create_browser(self.config['website'].get(),
self.config['login'].get(),
self.config['password'].get())
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
if not _id.isdigit():
raise AccountNotFound()
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
def iter_history(self, account):
for history in self.browser.get_history(account):
yield history
def transfer(self, account, to, amount, reason=None):
return self.browser.do_transfer(account, to, amount, reason)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/cragr/browser.py 0000664 0000000 0000000 00000025147 11666415431 0027043 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from weboob.capabilities.bank import Transfer, TransferError
from weboob.backends.cragr import pages
import mechanize
from datetime import datetime
import re
class Cragr(BaseBrowser):
PROTOCOL = 'https'
ENCODING = 'utf-8'
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
# a session id that is sometimes added, and should be ignored when matching pages
SESSION_REGEXP = '(?:|%s[A-Z0-9]+)' % re.escape(r';jsessionid=')
is_logging = False
def __init__(self, website, *args, **kwargs):
self.DOMAIN = website
self.PAGES = {'https://[^/]+/': pages.LoginPage,
'https://[^/]+/.*\.c.*': pages.AccountsList,
'https://[^/]+/login/process%s' % self.SESSION_REGEXP: pages.AccountsList,
'https://[^/]+/accounting/listAccounts': pages.AccountsList,
'https://[^/]+/accounting/listOperations': pages.AccountsList,
'https://[^/]+/accounting/showAccountDetail.+': pages.AccountsList,
'https://[^/]+/accounting/showMoreAccountOperations.*': pages.AccountsList,
}
BaseBrowser.__init__(self, *args, **kwargs)
def viewing_html(self):
"""
As the fucking HTTP server returns a document in unknown mimetype
'application/vnd.wap.xhtml+xml' it is not recognized by mechanize.
So this is a fucking hack.
"""
return True
def is_logged(self):
logged = self.page and self.page.is_logged() or self.is_logging
self.logger.debug('logged: %s' % (logged and 'yes' or 'no'))
return logged
def login(self):
"""
Attempt to log in.
Note: this method does nothing if we are already logged in.
"""
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
# Do we really need to login?
if self.is_logged():
self.logger.debug('already logged in')
return
self.is_logging = True
# Are we on the good page?
if not self.is_on_page(pages.LoginPage):
self.logger.debug('going to login page')
BaseBrowser.home(self)
self.logger.debug('attempting to log in')
self.page.login(self.username, self.password)
self.is_logging = False
if not self.is_logged():
raise BrowserIncorrectPassword()
def get_accounts_list(self):
self.logger.debug('accounts list required')
self.home()
return self.page.get_list()
def home(self):
"""
Ensure we are both logged and on the accounts list.
"""
self.logger.debug('accounts list page required')
if self.is_on_page(pages.AccountsList) and self.page.is_accounts_list():
self.logger.debug('already on accounts list')
return
# simply go to http(s)://the.doma.in/
BaseBrowser.home(self)
if self.is_on_page(pages.LoginPage):
if not self.is_logged():
# So, we are not logged on the login page -- what about logging ourselves?
self.login()
# we assume we are logged in
# for some regions, we may stay on the login page once we're
# logged in, without being redirected...
if self.is_on_page(pages.LoginPage):
# ... so we have to move by ourselves
self.move_to_accounts_list()
def move_to_accounts_list(self):
"""
For regions where you can stay on http(s)://the.doma.in/ while you are
logged in, move to the accounts list
"""
self.location('%s://%s/accounting/listAccounts' % (self.PROTOCOL, self.DOMAIN))
def get_account(self, id):
assert isinstance(id, basestring)
l = self.get_accounts_list()
for a in l:
if a.id == ('%s' % id):
return a
return None
def get_history(self, account):
history_url = account.link_id
operations_count = 0
# 1st, go on the account page
self.logger.debug('going on: %s' % history_url)
self.location('https://%s%s' % (self.DOMAIN, history_url))
# Some regions have a "Show more" (well, actually "Voir les 25
# suivants") link we have to use to get all the operations.
# However, it does not show only the 25 next results, it *adds* them
# to the current view. Therefore, we have to parse each new page using
# an offset, in order to ignore all already-fetched operations.
# This especially occurs on CA Centre.
use_expand_url = bool(self.page.expand_history_page_url())
while (history_url):
# we skip "operations_count" operations on each page if we are in the case described above
operations_offset = operations_count if use_expand_url else 0
for page_operation in self.page.get_history(operations_count, operations_offset):
operations_count += 1
yield page_operation
history_url = self.page.expand_history_page_url() if use_expand_url else self.page.next_page_url()
self.logger.debug('going on: %s' % history_url)
self.location('https://%s%s' % (self.DOMAIN, history_url))
def dict_find_value(self, dictionary, value):
"""
Returns the first key pointing on the given value, or None if none
is found.
"""
for k, v in dictionary.iteritems():
if v == value:
return k
return None
def do_transfer(self, account, to, amount, reason=None):
"""
Transfer the given amount of money from an account to another,
tagging the transfer with the given reason.
"""
# access the transfer page
transfer_page_unreachable_message = u'Could not reach the transfer page.'
self.home()
if not self.page.is_accounts_list():
raise TransferError(transfer_page_unreachable_message)
operations_url = self.page.operations_page_url()
self.location('https://%s%s' % (self.DOMAIN, operations_url))
transfer_url = self.page.transfer_page_url()
abs_transfer_url = 'https://%s%s' % (self.DOMAIN, transfer_url)
self.location(abs_transfer_url)
if not self.page.is_transfer_page():
raise TransferError(transfer_page_unreachable_message)
source_accounts = self.page.get_transfer_source_accounts()
target_accounts = self.page.get_transfer_target_accounts()
# check that the given source account can be used
if not account in source_accounts.values():
raise TransferError('You cannot use account %s as a source account.' % account)
# check that the given source account can be used
if not to in target_accounts.values():
raise TransferError('You cannot use account %s as a target account.' % to)
# separate euros from cents
amount_euros = int(amount)
amount_cents = int((amount * 100) - (amount_euros * 100))
# let's circumvent https://github.com/jjlee/mechanize/issues/closed#issue/17
# using http://wwwsearch.sourceforge.net/mechanize/faq.html#usage
adjusted_response = self.response().get_data().replace(' ', ' ')
response = mechanize.make_response(adjusted_response, [('Content-Type', 'text/html')], abs_transfer_url, 200, 'OK')
self.set_response(response)
# fill the form
self.select_form(nr=0)
self['numCompteEmetteur'] = ['%s' % self.dict_find_value(source_accounts, account)]
self['numCompteBeneficiaire'] = ['%s' % self.dict_find_value(target_accounts, to)]
self['montantPartieEntiere'] = '%s' % amount_euros
self['montantPartieDecimale'] = '%02d' % amount_cents
if reason != None:
self['libelle'] = reason
self.submit()
# look for known errors
content = unicode(self.response().get_data(), 'utf-8')
insufficient_amount_message = u'Montant insuffisant.'
maximum_allowed_balance_message = u'Solde maximum autorisé dépassé.'
if content.find(insufficient_amount_message) != -1:
raise TransferError('The amount you tried to transfer is too low.')
if content.find(maximum_allowed_balance_message) != -1:
raise TransferError('The maximum allowed balance for the target account has been / would be reached.')
# look for the known "all right" message
ready_for_transfer_message = u'Vous allez effectuer un virement'
if not content.find(ready_for_transfer_message):
raise TransferError('The expected message "%s" was not found.' % ready_for_transfer_message)
# submit the last form
self.select_form(nr=0)
submit_date = datetime.now()
self.submit()
# look for the known "everything went well" message
content = unicode(self.response().get_data(), 'utf-8')
transfer_ok_message = u'Vous venez d\'effectuer un virement du compte'
if not content.find(transfer_ok_message):
raise TransferError('The expected message "%s" was not found.' % transfer_ok_message)
# We now have to return a Transfer object
# the final page does not provide any transfer id, so we'll use the submit date
transfer = Transfer(submit_date.strftime('%Y%m%d%H%M%S'))
transfer.amount = amount
transfer.origin = account
transfer.recipient = to
transfer.date = submit_date
return transfer
#def get_coming_operations(self, account):
# if not self.is_on_page(pages.AccountComing) or self.page.account.id != account.id:
# self.location('/NS_AVEEC?ch4=%s' % account.link_id)
# return self.page.get_operations()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/cragr/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026074 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/cragr/pages/__init__.py 0000664 0000000 0000000 00000001516 11666415431 0030210 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .accounts_list import AccountsList
from .login import LoginPage
__all__ = ['AccountsList', 'LoginPage']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/cragr/pages/accounts_list.py 0000664 0000000 0000000 00000027746 11666415431 0031340 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from weboob.capabilities.bank import Account
from .base import CragrBasePage
from weboob.capabilities.bank import Operation
def clean_amount(amount):
"""
Removes weird characters and converts to a float
>>> clean_amount(u'1 000,00 $')
1000.0
"""
data = amount.replace(',', '.').replace(' ', '').replace(u'\xa0', '')
matches = re.findall('^(-?[0-9]+\.[0-9]{2}).*$', data)
return float(matches[0]) if (matches) else 0.0
class AccountsList(CragrBasePage):
def get_list(self):
"""
Returns the list of available bank accounts
"""
l = []
for div in self.document.getiterator('div'):
if div.attrib.get('class', '') == 'dv' and div.getchildren()[0].tag in ('a', 'br'):
account = Account()
if div.getchildren()[0].tag == 'a':
# This is at least present on CA Nord-Est
account.label = ' '.join(div.find('a').text.split()[:-1])
account.link_id = div.find('a').get('href', '')
account.id = div.find('a').text.split()[-1]
s = div.find('div').find('b').find('span').text
else:
# This is at least present on CA Toulouse
account.label = div.find('a').text.strip()
account.link_id = div.find('a').get('href', '')
account.id = div.findall('br')[1].tail.strip()
s = div.find('div').find('b').text
account.balance = clean_amount(s)
l.append(account)
return l
def is_accounts_list(self):
"""
Returns True if the current page appears to be the page dedicated to
list the accounts.
"""
# we check for the presence of a "mes comptes titres" link_id
link = self.document.xpath('/html/body//a[contains(text(), "comptes titres")]')
return bool(link)
def is_account_page(self):
"""
Returns True if the current page appears to be a page dedicated to list
the history of a specific account.
"""
# tested on CA Lorraine, Paris, Toulouse
title_spans = self.document.xpath('/html/body//div[@class="dv"]/span')
for title_span in title_spans:
title_text = title_span.text_content().strip().replace("\n", '')
if (re.match('.*Compte.*n.*[0-9]+.*au.*', title_text)):
return True
return False
def is_transfer_page(self):
"""
Returns True if the current page appears to be the page dedicated to
order transfers between accounts.
"""
source_account_select_field = self.document.xpath('/html/body//form//select[@name="numCompteEmetteur"]')
target_account_select_field = self.document.xpath('/html/body//form//select[@name="numCompteBeneficiaire"]')
return bool(source_account_select_field) and bool(target_account_select_field)
def get_transfer_accounts(self, select_name):
"""
Returns the accounts proposed for a transfer in a select field.
This method assumes the current page is the one dedicated to transfers.
select_name is the name of the select field to analyze
"""
if not self.is_transfer_page():
return False
source_accounts = {}
source_account_options = self.document.xpath('/html/body//form//select[@name="%s"]/option' % select_name)
for option in source_account_options:
source_account_value = option.get('value', -1)
if (source_account_value != -1):
matches = re.findall('^[A-Z0-9]+.*([0-9]{11}).*$', self.extract_text(option))
if matches:
source_accounts[source_account_value] = matches[0]
return source_accounts
def get_transfer_source_accounts(self):
return self.get_transfer_accounts('numCompteEmetteur')
def get_transfer_target_accounts(self):
return self.get_transfer_accounts('numCompteBeneficiaire')
def expand_history_page_url(self):
"""
When on a page dedicated to list the history of a specific account (see
is_account_page), returns the link to expand the history with 25 more results,
or False if the link is not present.
"""
# tested on CA centre france
a = self.document.xpath('/html/body//div[@class="navlink"]//a[contains(text(), "Voir les 25 suivants")]')
if not a:
return False
else:
return a[0].get('href', '')
def next_page_url(self):
"""
When on a page dedicated to list the history of a specific account (see
is_account_page), returns the link to the next page, or False if the
link is not present.
"""
# tested on CA Lorraine, Paris, Toulouse
a = self.document.xpath('/html/body//div[@class="navlink"]//a[contains(text(), "Suite")]')
if not a:
return False
else:
return a[0].get('href', '')
def operations_page_url(self):
"""
Returns the link to the "Opérations" page. This function assumes the
current page is the accounts list (see is_accounts_list)
"""
link = self.document.xpath(u'/html/body//a[contains(text(), "Opérations")]')
return link[0].get('href')
def transfer_page_url(self):
"""
Returns the link to the "Virements" page. This function assumes the
current page is the operations list (see operations_page_url)
"""
link = self.document.xpath('/html/body//a[@accesskey=1]/@href')
return link[0]
def is_right_aligned_div(self, div_elmt):
"""
Returns True if the given div element is right-aligned
"""
return(re.match('.*text-align: ?right.*', div_elmt.get('style', '')))
def extract_text(self, xml_elmt):
"""
Given an XML element, returns its inner text in a reasonably readable way
"""
data = u''
for text in xml_elmt.itertext():
data = data + u'%s ' % text
data = re.sub(' +', ' ', data.replace("\n", ' ').strip())
return data
def get_history(self, start_index = 0, start_offset = 0):
"""
Returns the history of a specific account. Note that this function
expects the current page to be the one dedicated to this history.
start_index is the id used for the first created operation.
start_offset allows ignoring the `n' first Operations on the page.
"""
# tested on CA Lorraine, Paris, Toulouse
# avoir parsing the page as an account-dedicated page if it is not the case
if not self.is_account_page():
return
index = start_index
operation = False
skipped = 0
body_elmt_list = self.document.xpath('/html/body/*')
# type of separator used in the page
separators = 'hr'
# How many elements do we have under the ?
sep_expected = len(self.document.xpath('/html/body/hr'))
if (not sep_expected):
# no ? Then how many class-less
used as separators instead?
sep_expected = len(self.document.xpath('/html/body/div[not(@class) and not(@style)]'))
separators = 'div'
# the interesting divs are after the elements
interesting_divs = []
right_div_count = 0
left_div_count = 0
sep_found = 0
for body_elmt in body_elmt_list:
if (separators == 'hr' and body_elmt.tag == 'hr'):
sep_found += 1
elif (separators == 'div' and body_elmt.tag == 'div' and body_elmt.get('class', 'nope') == 'nope'):
sep_found += 1
elif (sep_found >= sep_expected and body_elmt.tag == 'div'):
# we just want
with dv class and a style attribute
if (body_elmt.get('class', '') != 'dv'):
continue
if (body_elmt.get('style', 'nope') == 'nope'):
continue
interesting_divs.append(body_elmt)
if (self.is_right_aligned_div(body_elmt)):
right_div_count += 1
else:
left_div_count += 1
# new layout that is somewhat easier to parse (found at Toulouse)
table_layout = len(self.document.xpath("id('operationsHeader')")) > 0
# So, how are data laid out?
alternate_layout = (left_div_count == 2 * right_div_count)
# we'll have: one left-aligned div for the date, one right-aligned
# div for the amount, and one left-aligned div for the label. Each time.
if table_layout:
lines = self.document.xpath('id("operationsContent")//table[@class="tb"]/tr')
for line in lines:
if skipped < start_offset:
skipped += 1
continue
operation = Operation(index)
index += 1
operation.date = self.extract_text(line[0])
operation.label = self.extract_text(line[1])
operation.amount = clean_amount(self.extract_text(line[2]))
yield operation
elif (not alternate_layout):
for body_elmt in interesting_divs:
if skipped < start_offset:
if self.is_right_aligned_div(body_elmt):
skipped += 1
continue
if (self.is_right_aligned_div(body_elmt)):
# this is the second line of an operation entry, displaying the amount
operation.amount = clean_amount(self.extract_text(body_elmt))
yield operation
else:
# this is the first line of an operation entry, displaying the date and label
data = self.extract_text(body_elmt)
matches = re.findall('^([012][0-9]|3[01])/(0[1-9]|1[012]).(.+)$', data)
operation = Operation(index)
index += 1
if (matches):
operation.date = u'%s/%s' % (matches[0][0], matches[0][1])
operation.label = u'%s' % matches[0][2]
else:
operation.date = u'01/01'
operation.label = u'Unknown'
else:
for i in range(0, len(interesting_divs)/3):
if skipped < start_offset:
skipped += 1
continue
operation = Operation(index)
index += 1
# amount
operation.amount = clean_amount(self.extract_text(interesting_divs[(i*3)+1]))
# date
data = self.extract_text(interesting_divs[i*3])
matches = re.findall('^([012][0-9]|3[01])/(0[1-9]|1[012])', data)
operation.date = u'%s/%s' % (matches[0][0], matches[0][1]) if (matches) else u'01/01'
#label
data = self.extract_text(interesting_divs[(i*3)+2])
data = re.sub(' +', ' ', data)
operation.label = u'%s' % data
yield operation
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/cragr/pages/base.py 0000664 0000000 0000000 00000003543 11666415431 0027365 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage
from weboob.tools.browser import BrowserUnavailable
class CragrBasePage(BasePage):
def on_loaded(self):
# Check for an error
for div in self.document.getiterator('div'):
if div.attrib.get('class', '') == 'dv' and div.getchildren()[0].tag in ('img') and div.getchildren()[0].attrib.get('alt', '') == 'Attention':
# Try to find a detailed error message
if div.getchildren()[1].tag == 'span':
raise BrowserUnavailable(div.find('span').find('b').text)
elif div.getchildren()[1].tag == 'b':
# I haven't encountered this variation in the wild,
# but I wouldn't be surprised if it existed
# given the similar differences between regions.
raise BrowserUnavailable(div.find('b').find('span').text)
raise BrowserUnavailable()
def is_logged(self):
return not self.document.xpath('/html/body//form//input[@name = "code"]') and \
not self.document.xpath('/html/body//form//input[@name = "userPassword"]')
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/cragr/pages/login.py 0000664 0000000 0000000 00000003723 11666415431 0027563 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.mech import ClientForm
ControlNotFoundError = ClientForm.ControlNotFoundError
from .base import CragrBasePage
__all__ = ['LoginPage']
class LoginPage(CragrBasePage):
def login(self, login, password):
self.browser.select_form(nr=0)
try:
self.browser['numero'] = login
self.browser['code'] = password
except ControlNotFoundError:
try:
self.browser['userLogin'] = login
self.browser['userPassword'] = password
except ControlNotFoundError:
self.browser.controls.append(ClientForm.TextControl('text', 'numero', {'value': ''}))
self.browser.controls.append(ClientForm.TextControl('text', 'code', {'value': ''}))
self.browser.controls.append(ClientForm.TextControl('text', 'userLogin', {'value': ''}))
self.browser.controls.append(ClientForm.TextControl('text', 'userPassword', {'value': ''}))
self.browser.set_all_readonly(False)
self.browser['numero'] = login
self.browser['code'] = password
self.browser['userLogin'] = login
self.browser['userPassword'] = password
self.browser.submit()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/cragr/test.py 0000664 0000000 0000000 00000001744 11666415431 0026334 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class CrAgrTest(BackendTest):
BACKEND = 'cragr'
def test_cragr(self):
l = list(self.backend.iter_accounts())
if len(l) > 0:
a = l[0]
list(self.backend.iter_history(a))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/creditmutuel/ 0000775 0000000 0000000 00000000000 11666415431 0026405 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/creditmutuel/__init__.py 0000664 0000000 0000000 00000001456 11666415431 0030524 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import CreditMutuelBackend
__all__ = ['CreditMutuelBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/creditmutuel/backend.py 0000664 0000000 0000000 00000004222 11666415431 0030346 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.bank import ICapBank, AccountNotFound
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import ValueBackendPassword
from .browser import CreditMutuelBrowser
__all__ = ['CreditMutuelBackend']
class CreditMutuelBackend(BaseBackend, ICapBank):
NAME = 'creditmutuel'
MAINTAINER = 'Julien Veyssier'
EMAIL = 'julien.veyssier@aiur.fr'
VERSION = '0.9.1'
DESCRIPTION = u'Crédit Mutuel french bank'
LICENSE = 'AGPLv3+'
CONFIG = BackendConfig(ValueBackendPassword('login', label='Account ID', regexp='^\d{1,13}\w$', masked=False),
ValueBackendPassword('password', label='Password of account'))
BROWSER = CreditMutuelBrowser
def create_default_browser(self):
return self.create_browser(self.config['login'].get(), self.config['password'].get())
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
if not _id.isdigit():
raise AccountNotFound()
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
def iter_operations(self, account):
""" TODO Not supported yet """
return iter([])
def iter_history(self, account):
for history in self.browser.get_history(account):
yield history
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/creditmutuel/browser.py 0000664 0000000 0000000 00000010351 11666415431 0030442 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from .pages import LoginPage, LoginErrorPage, AccountsPage, OperationsPage, InfoPage
__all__ = ['CreditMutuelBrowser']
# Browser
class CreditMutuelBrowser(BaseBrowser):
PROTOCOL = 'https'
DOMAIN = 'www.creditmutuel.fr'
ENCODING = 'iso-8859-1'
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
PAGES = {'https://www.creditmutuel.fr/groupe/fr/index.html': LoginPage,
'https://www.creditmutuel.fr/groupe/fr/identification/default.cgi': LoginErrorPage,
'https://www.creditmutuel.fr/.*/fr/banque/situation_financiere.cgi': AccountsPage,
'https://www.creditmutuel.fr/.*/fr/banque/mouvements.cgi.*' : OperationsPage,
'https://www.creditmutuel.fr/.*/fr/banque/BAD.*' : InfoPage
}
def __init__(self, *args, **kwargs):
BaseBrowser.__init__(self, *args, **kwargs)
#self.SUB_BANKS = ['cmdv','cmcee','cmse', 'cmidf', 'cmsmb', 'cmma', 'cmmabn', 'cmc', 'cmlaco', 'cmnormandie', 'cmm']
#self.currentSubBank = None
def is_logged(self):
return self.page and not self.is_on_page(LoginPage)
def home(self):
return self.location('https://www.creditmutuel.fr/groupe/fr/index.html')
def login(self):
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
if not self.is_on_page(LoginPage):
self.location('https://www.creditmutuel.fr/', no_login=True)
self.page.login( self.username, self.password)
if not self.is_logged() or self.is_on_page(LoginErrorPage):
raise BrowserIncorrectPassword()
self.SUB_BANKS = ['cmdv','cmcee','cmse', 'cmidf', 'cmsmb', 'cmma', 'cmmabn', 'cmc', 'cmlaco', 'cmnormandie', 'cmm']
self.getCurrentSubBank()
def get_accounts_list(self):
if not self.is_on_page(AccountsPage):
self.location('https://www.creditmutuel.fr/%s/fr/banque/situation_financiere.cgi'%self.currentSubBank)
return self.page.get_list()
def get_account(self, id):
assert isinstance(id, basestring)
l = self.get_accounts_list()
for a in l:
if a.id == id:
return a
return None
def getCurrentSubBank(self):
# the account list and history urls depend on the sub bank of the user
current_url = self.geturl()
current_url_parts = current_url.split('/')
for subbank in self.SUB_BANKS:
if subbank in current_url_parts:
self.currentSubBank = subbank
def get_history(self, account):
page_url = account.link_id
#operations_count = 0
l_ret = []
while (page_url):
self.location('https://%s/%s/fr/banque/%s' % (self.DOMAIN, self.currentSubBank, page_url))
#for page_operation in self.page.get_history(operations_count):
# operations_count += 1
# yield page_operation
## FONCTIONNE
#for op in self.page.get_history():
# yield op
## FONTIONNE
#return self.page.get_history()
for op in self.page.get_history():
l_ret.append(op)
page_url = self.page.next_page_url()
return l_ret
#def get_coming_operations(self, account):
# if not self.is_on_page(AccountComing) or self.page.account.id != account.id:
# self.location('/NS_AVEEC?ch4=%s' % account.link_id)
# return self.page.get_operations()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/creditmutuel/pages.py 0000664 0000000 0000000 00000006613 11666415431 0030064 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage
from weboob.capabilities.bank import Account
from weboob.capabilities.bank import Operation
class LoginPage(BasePage):
def login(self, login, passwd):
self.browser.select_form(nr=0)
self.browser['_cm_user'] = login
self.browser['_cm_pwd'] = passwd
self.browser.submit()
class LoginErrorPage(BasePage):
pass
class InfoPage(BasePage):
pass
class AccountsPage(BasePage):
def get_list(self):
l = []
for tr in self.document.getiterator('tr'):
first_td = tr.getchildren()[0]
if first_td.attrib.get('class', '') == 'i g' or first_td.attrib.get('class', '') == 'p g':
account = Account()
account.label = u"%s"%first_td.find('a').text
account.link_id = first_td.find('a').get('href', '')
account.id = first_td.find('a').text.split(' ')[0]+first_td.find('a').text.split(' ')[1]
s = tr.getchildren()[2].text
if s.strip() == "":
s = tr.getchildren()[1].text
balance = u''
for c in s:
if c.isdigit() or c == '-':
balance += c
if c == ',':
balance += '.'
account.balance = float(balance)
l.append(account)
#raise NotImplementedError()
return l
def next_page_url(self):
""" TODO pouvoir passer à la page des comptes suivante """
return 0
class OperationsPage(BasePage):
def get_history(self):
index = 0
for tr in self.document.getiterator('tr'):
first_td = tr.getchildren()[0]
if first_td.attrib.get('class', '') == 'i g' or first_td.attrib.get('class', '') == 'p g':
operation = Operation(index)
index += 1
operation.date = first_td.text
operation.label = u"%s"%tr.getchildren()[2].text.replace('\n',' ')
if len(tr.getchildren()[3].text) > 2:
s = tr.getchildren()[3].text
elif len(tr.getchildren()[4].text) > 2:
s = tr.getchildren()[4].text
else:
s = "0"
balance = u''
for c in s:
if c.isdigit() or c == "-":
balance += c
if c == ',':
balance += '.'
operation.amount = float(balance)
yield operation
def next_page_url(self):
""" TODO pouvoir passer à la page des opérations suivantes """
return 0
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/creditmutuel/test.py 0000664 0000000 0000000 00000001616 11666415431 0027742 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class CreditMutuelTest(BackendTest):
BACKEND = 'crmut'
def test_crmut(self):
list(self.backend.iter_accounts())
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dailymotion/ 0000775 0000000 0000000 00000000000 11666415431 0026227 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dailymotion/__init__.py 0000664 0000000 0000000 00000000112 11666415431 0030332 0 ustar 00root root 0000000 0000000 from .backend import DailymotionBackend
__all__ = ['DailymotionBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dailymotion/backend.py 0000664 0000000 0000000 00000004140 11666415431 0030167 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.video import ICapVideo
from weboob.tools.backend import BaseBackend
from .browser import DailymotionBrowser
from .video import DailymotionVideo
__all__ = ['DailymotionBackend']
class DailymotionBackend(BaseBackend, ICapVideo):
NAME = 'dailymotion'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
DESCRIPTION = 'Dailymotion videos website'
LICENSE = 'AGPLv3+'
BROWSER = DailymotionBrowser
def get_video(self, _id):
with self.browser:
return self.browser.get_video(_id)
SORTBY = ['relevance', 'rated', 'visited', None]
def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
with self.browser:
return self.browser.iter_search_results(pattern, self.SORTBY[sortby])
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
with self.browser:
video = self.browser.get_video(DailymotionVideo.id2url(video.id), video)
if 'thumbnail' in fields and video.thumbnail:
with self.browser:
video.thumbnail.data = self.browser.readurl(video.thumbnail.url)
return video
OBJECTS = {DailymotionVideo: fill_video}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dailymotion/browser.py 0000664 0000000 0000000 00000003523 11666415431 0030267 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from urllib import quote_plus
from weboob.tools.browser import BaseBrowser
from weboob.tools.browser.decorators import id2url
from .pages import IndexPage, VideoPage
from .video import DailymotionVideo
__all__ = ['DailymotionBrowser']
class DailymotionBrowser(BaseBrowser):
DOMAIN = 'dailymotion.com'
ENCODING = None
PAGES = {r'http://[w\.]*dailymotion\.com/?': IndexPage,
r'http://[w\.]*dailymotion\.com/(\w+/)?search/.*': IndexPage,
r'http://[w\.]*dailymotion\.com/video/(?P.+)': VideoPage,
}
@id2url(DailymotionVideo.id2url)
def get_video(self, url, video=None):
self.location(url)
return self.page.get_video(video)
def iter_search_results(self, pattern, sortby):
if not pattern:
self.home()
else:
if sortby is None:
url = '/search/%s/1' % quote_plus(pattern.encode('utf-8'))
else:
url = '/%s/search/%s/1' % (sortby, quote_plus(pattern.encode('utf-8')))
self.location(url)
assert self.is_on_page(IndexPage)
return self.page.iter_videos()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dailymotion/pages.py 0000664 0000000 0000000 00000010640 11666415431 0027701 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import datetime
import urllib
import re
from weboob.tools.capabilities.thumbnail import Thumbnail
from weboob.capabilities.base import NotAvailable
from weboob.tools.misc import html2text
from weboob.tools.browser import BasePage, BrokenPageError
from .video import DailymotionVideo
__all__ = ['IndexPage', 'VideoPage']
class IndexPage(BasePage):
def iter_videos(self):
for div in self.parser.select(self.document.getroot(), 'div.dmpi_video_item'):
_id = 0
for cls in div.attrib['class'].split():
if cls.startswith('id_'):
_id = int(cls[3:])
break
if _id == 0:
self.browser.logger.warning('Unable to find the ID of a video')
continue
video = DailymotionVideo(int(_id))
video.title = self.parser.select(div, 'h3 a', 1).text
video.author = self.parser.select(div, 'div.dmpi_user_login', 1).find('a').text
video.description = html2text(self.parser.tostring(self.parser.select(div, 'div.dmpi_video_description', 1))).strip()
try:
parts = self.parser.select(div, 'div.duration', 1).text.split(':')
except BrokenPageError:
# it's probably a live, np.
video.duration = NotAvailable
else:
if len(parts) == 1:
seconds = parts[0]
hours = minutes = 0
elif len(parts) == 2:
minutes, seconds = parts
hours = 0
elif len(parts) == 3:
hours, minutes, seconds = parts
else:
raise BrokenPageError('Unable to parse duration %r' % self.parser.select(div, 'div.duration', 1).text)
video.duration = datetime.timedelta(hours=int(hours), minutes=int(minutes), seconds=int(seconds))
url = self.parser.select(div, 'img.dmco_image', 1).attrib['src']
video.thumbnail = Thumbnail(url)
rating_div = self.parser.select(div, 'div.small_stars', 1)
video.rating_max = self.get_rate(rating_div)
video.rating = self.get_rate(rating_div.find('div'))
# XXX missing date
video.date = NotAvailable
yield video
def get_rate(self, div):
m = re.match('width: *(\d+)px', div.attrib['style'])
if m:
return int(m.group(1))
else:
self.browser.logger.warning('Unable to parse rating: %s' % div.attrib['style'])
return 0
class VideoPage(BasePage):
def get_video(self, video=None):
if video is None:
video = DailymotionVideo(self.group_dict['id'])
div = self.parser.select(self.document.getroot(), 'div#content', 1)
video.title = self.parser.select(div, 'span.title', 1).text
video.author = self.parser.select(div, 'a.name', 1).text
try:
video.description = self.parser.select(div, 'div#video_description', 1).text
except BrokenPageError:
video.description = u''
for script in self.parser.select(self.document.getroot(), 'div.dmco_html'):
if 'id' in script.attrib and script.attrib['id'].startswith('container_player_'):
text = script.find('script').text
mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', text)
if mobj is None:
mobj = re.search('"sdURL":.*?"(.*?)"', urllib.unquote(text))
mediaURL = mobj.group(1).replace("\\", "")
else:
mediaURL = urllib.unquote(mobj.group(1))
video.url = mediaURL
return video
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dailymotion/test.py 0000664 0000000 0000000 00000002250 11666415431 0027557 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class DailymotionTest(BackendTest):
BACKEND = 'dailymotion'
def test_dailymotion(self):
l = list(self.backend.iter_search_results('chirac'))
self.assertTrue(len(l) > 0)
v = l[0]
self.backend.fillobj(v, ('url',))
self.assertTrue(v.url and v.url.startswith('http://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
self.backend.browser.openurl(v.url)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dailymotion/video.py 0000664 0000000 0000000 00000002055 11666415431 0027711 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.video import BaseVideo
__all__ = ['DailymotionVideo']
class DailymotionVideo(BaseVideo):
def __init__(self, *args, **kwargs):
BaseVideo.__init__(self, *args, **kwargs)
self.ext = 'flv'
@classmethod
def id2url(cls, _id):
return 'http://www.dailymotion.com/video/%s' % _id
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dlfp/ 0000775 0000000 0000000 00000000000 11666415431 0024624 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dlfp/__init__.py 0000664 0000000 0000000 00000001476 11666415431 0026745 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .browser import DLFP
from .backend import DLFPBackend
__all__ = ['DLFP', 'DLFPBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dlfp/backend.py 0000664 0000000 0000000 00000017203 11666415431 0026570 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.newsfeed import Newsfeed
from weboob.tools.value import Value, ValueBool, ValueBackendPassword
from weboob.tools.misc import limit
from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Message, Thread, CantSendMessage
from weboob.capabilities.content import ICapContent, Content
from .browser import DLFP
from .tools import rssid, id2url
__all__ = ['DLFPBackend']
class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapContent):
NAME = 'dlfp'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = "Da Linux French Page"
CONFIG = BackendConfig(Value('username', label='Username', regexp='.+'),
ValueBackendPassword('password', label='Password'),
ValueBool('get_news', label='Get newspapers', default=True),
ValueBool('get_diaries', label='Get diaries', default=False),
ValueBool('get_polls', label='Get polls', default=False),
ValueBool('get_board', label='Get board', default=False),
ValueBool('get_wiki', label='Get wiki', default=False),
ValueBool('get_tracker', label='Get tracker', default=False))
STORAGE = {'seen': {}}
BROWSER = DLFP
FEEDS = {'get_news': "https://linuxfr.org/news.atom",
'get_diaries': "https://linuxfr.org/journaux.atom",
'get_polls': "https://linuxfr.org/sondages.atom",
'get_board': "https://linuxfr.org/forums.atom",
'get_wiki': "https://linuxfr.org/wiki.atom",
'get_tracker': "https://linuxfr.org/suivi.atom",
}
def create_default_browser(self):
return self.create_browser(self.config['username'].get(), self.config['password'].get())
def deinit(self):
# don't need to logout if the browser hasn't been used.
if not self._browser:
return
with self.browser:
self.browser.close_session()
#### ICapMessages ##############################################
def iter_threads(self):
whats = set()
for param, url in self.FEEDS.iteritems():
if self.config[param].get():
whats.add(url)
for what in whats:
for article in limit(Newsfeed(what, rssid).iter_entries(), 20):
thread = Thread(article.id)
thread.title = article.title
if article.datetime:
thread.date = article.datetime
yield thread
def get_thread(self, id):
if isinstance(id, Thread):
thread = id
id = thread.id
else:
thread = None
with self.browser:
content = self.browser.get_content(id)
if not content:
return None
if not thread:
thread = Thread(content.id)
flags = Message.IS_HTML
if not thread.id in self.storage.get('seen', default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(thread=thread,
id=0, # root message
title=content.title,
sender=content.author or u'',
receivers=None,
date=thread.date,
parent=None,
content=content.body,
signature='URL: %s' % self.browser.absurl(id2url(content.id)),
children=[],
flags=flags)
for com in content.comments:
self._insert_comment(com, thread.root)
return thread
def _insert_comment(self, com, parent):
""""
Insert 'com' comment and its children in the parent message.
"""
flags = Message.IS_HTML
if not com.id in self.storage.get('seen', parent.thread.id, 'comments', default=[]):
flags |= Message.IS_UNREAD
message = Message(thread=parent.thread,
id=com.id,
title=com.title,
sender=com.author or u'',
receivers=None,
date=com.date,
parent=parent,
content=com.body,
signature=com.signature + \
' '.join(['Score: %d' % com.score,
'URL: %s' % com.url]),
children=[],
flags=flags)
parent.children.append(message)
for sub in com.comments:
self._insert_comment(sub, message)
def iter_unread_messages(self, thread=None):
for thread in self.iter_threads():
self.fill_thread(thread, 'root')
for m in thread.iter_all_messages():
if m.flags & m.IS_UNREAD:
yield m
def set_message_read(self, message):
self.storage.set('seen', message.thread.id, 'comments',
self.storage.get('seen', message.thread.id, 'comments', default=[]) + [message.id])
self.storage.save()
def fill_thread(self, thread, fields):
return self.get_thread(thread)
#### ICapMessagesReply #########################################
def post_message(self, message):
if not message.parent:
raise CantSendMessage('Posting news and diaries on DLFP is not supported yet')
assert message.thread
with self.browser:
return self.browser.post_comment(message.thread.id,
message.parent.id,
message.title,
message.content)
#### ICapContent ###############################################
def get_content(self, id):
if isinstance(id, basestring):
content = Content(id)
else:
content = id
id = content.id
with self.browser:
data = self.browser.get_wiki_content(id)
if data is None:
return None
content.content = data
return content
def push_content(self, content, message=None, minor=False):
with self.browser:
return self.browser.set_wiki_content(content.id, content.content, message)
def get_content_preview(self, content):
with self.browser:
return self.browser.get_wiki_preview(content.id, content.content)
OBJECTS = {Thread: fill_thread}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dlfp/browser.py 0000664 0000000 0000000 00000020211 11666415431 0026655 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import urllib
import re
from weboob.tools.browser import BaseBrowser, BrowserHTTPNotFound, BrowserHTTPError, BrowserIncorrectPassword
from weboob.capabilities.messages import CantSendMessage
from .pages.index import IndexPage, LoginPage
from .pages.news import ContentPage, NewCommentPage, NodePage, CommentPage, NewTagPage
from .pages.board import BoardIndexPage
from .pages.wiki import WikiEditPage
from .tools import id2url, url2id
# Browser
class DLFP(BaseBrowser):
DOMAIN = 'linuxfr.org'
PROTOCOL = 'https'
PAGES = {'https?://.*linuxfr.org/?': IndexPage,
'https?://.*linuxfr.org/compte/connexion': LoginPage,
'https?://.*linuxfr.org/news/[^\.]+': ContentPage,
'https?://.*linuxfr.org/wiki/(?!nouveau)[^/]+': ContentPage,
'https?://.*linuxfr.org/wiki': WikiEditPage,
'https?://.*linuxfr.org/wiki/nouveau': WikiEditPage,
'https?://.*linuxfr.org/wiki/[^\.]+/modifier': WikiEditPage,
'https?://.*linuxfr.org/suivi/[^\.]+': ContentPage,
'https?://.*linuxfr.org/sondages/[^\.]+': ContentPage,
'https?://.*linuxfr.org/users/[^\./]+/journaux/[^\.]+': ContentPage,
'https?://.*linuxfr.org/forums/[^\./]+/posts/[^\.]+': ContentPage,
'https?://.*linuxfr.org/nodes/(\d+)/comments/(\d+)': CommentPage,
'https?://.*linuxfr.org/nodes/(\d+)/comments/nouveau': NewCommentPage,
'https?://.*linuxfr.org/nodes/(\d+)/comments': NodePage,
'https?://.*linuxfr.org/nodes/(\d+)/tags/nouveau': NewTagPage,
'https?://.*linuxfr.org/board/index.xml': BoardIndexPage,
}
last_board_msg_id = None
def parse_id(self, _id):
if re.match('^https?://.*linuxfr.org/nodes/\d+/comments/\d+$', _id):
return _id, None
url = id2url(_id)
if url is None:
if url2id(_id) is not None:
url = _id
_id = url2id(url)
else:
return None, None
return url, _id
def get_wiki_content(self, _id):
url, _id = self.parse_id('W.%s' % _id)
if url is None:
return None
try:
self.location('%s/modifier' % url)
except BrowserHTTPNotFound:
return ''
assert self.is_on_page(WikiEditPage)
return self.page.get_body()
def _go_on_wiki_edit_page(self, name):
"""
Go on the wiki page named 'name'.
Return True if this is a new page, or False if
the page already exist.
Return None if it isn't a right wiki page name.
"""
url, _id = self.parse_id('W.%s' % name)
if url is None:
return None
try:
self.location('%s/modifier' % url)
except BrowserHTTPNotFound:
self.location('/wiki/nouveau')
new = True
else:
new = False
assert self.is_on_page(WikiEditPage)
return new
def set_wiki_content(self, name, content, message):
new = self._go_on_wiki_edit_page(name)
if new is None:
return None
if new:
title = name.replace('-', ' ')
else:
title = None
self.page.post_content(title, content, message)
def get_wiki_preview(self, name, content):
if self._go_on_wiki_edit_page(name) is None:
return None
self.page.post_preview(content)
if self.is_on_page(WikiEditPage):
return self.page.get_preview_html()
elif self.is_on_page(ContentPage):
return self.page.get_article().body
def get_content(self, _id):
url, _id = self.parse_id(_id)
if url is None:
return None
self.location(url)
self.page.url = self.absurl(url)
if self.is_on_page(CommentPage):
content = self.page.get_comment()
elif self.is_on_page(ContentPage):
m = re.match('.*#comment-(\d+)$', url)
if m:
content = self.page.get_comment(int(m.group(1)))
else:
content = self.page.get_article()
if _id is not None:
content.id = _id
return content
def _is_comment_submit_form(self, form):
return 'comment_new' in form.action
def post_comment(self, thread, reply_id, title, message):
url = id2url(thread)
if url is None:
raise CantSendMessage('%s is not a right ID' % thread)
self.location(url)
assert self.is_on_page(ContentPage)
self.location(self.page.get_post_comment_url())
assert self.is_on_page(NewCommentPage)
self.select_form(predicate=self._is_comment_submit_form)
self.set_all_readonly(False)
if title is not None:
self['comment[title]'] = title.encode('utf-8')
self['comment[wiki_body]'] = message.encode('utf-8')
if int(reply_id) > 0:
self['comment[parent_id]'] = str(reply_id)
self['commit'] = 'Poster le commentaire'
try:
self.submit()
except BrowserHTTPError, e:
raise CantSendMessage('Unable to send message to %s.%s: %s' % (thread, reply_id, e))
if self.is_on_page(NodePage):
errors = self.page.get_errors()
if len(errors) > 0:
raise CantSendMessage('Unable to send message: %s' % ', '.join(errors))
return None
def login(self):
# not usefull for the moment
#self.location('/', no_login=True)
data = {'account[login]': self.username,
'account[password]': self.password,
'account[remember_me]': 1,
#'authenticity_token': self.page.get_login_token(),
}
self.location('/compte/connexion', urllib.urlencode(data), no_login=True)
if not self.is_logged():
raise BrowserIncorrectPassword()
def is_logged(self):
return (self.page and self.page.is_logged())
def close_session(self):
self.openurl('/compte/deconnexion', {})
def plusse(self, url):
return self.relevance(url, 'for')
def moinse(self, url):
return self.relevance(url, 'against')
def relevance(self, url, what):
comment = self.get_content(url)
if comment is None:
raise ValueError('The given URL isn\'t a comment.')
if comment.relevance_token is None:
return False
res = self.readurl('%s%s' % (comment.relevance_url, what),
urllib.urlencode({'authenticity_token': comment.relevance_token}))
return res
def iter_new_board_messages(self):
self.location('/board/index.xml')
assert self.is_on_page(BoardIndexPage)
msgs = self.page.get_messages(self.last_board_msg_id)
for msg in reversed(msgs):
self.last_board_msg_id = msg.id
yield msg
def board_post(self, msg):
request = self.request_class(self.absurl('/board/'),
urllib.urlencode({'board[message]': msg}),
{'Referer': self.absurl('/')})
self.readurl(request)
def add_tag(self, _id, tag):
url, _id = self.parse_id(_id)
if url is None:
return None
self.location(url)
assert self.is_on_page(ContentPage)
self.location(self.page.get_tag_url())
assert self.is_on_page(NewTagPage)
self.page.tag(tag)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dlfp/pages/ 0000775 0000000 0000000 00000000000 11666415431 0025723 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dlfp/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0030022 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dlfp/pages/board.py 0000664 0000000 0000000 00000004073 11666415431 0027370 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from logging import warning
from weboob.tools.browser import BasePage
class Message(object):
TIMESTAMP_REGEXP = re.compile(r'(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})')
def __init__(self, id, timestamp, login, message, is_me):
self.id = id
self.timestamp = timestamp
self.login = login
self.message = message
self.is_me = is_me
self.norloge = timestamp
m = self.TIMESTAMP_REGEXP.match(timestamp)
if m:
self.norloge = '%02d:%02d:%02d' % (int(m.group(4)),
int(m.group(5)),
int(m.group(6)))
else:
warning('Unable to parse timestamp "%s"' % timestamp)
class BoardIndexPage(BasePage):
def is_logged(self):
return True
def get_messages(self, last=None):
msgs = []
for post in self.parser.select(self.document.getroot(), 'post'):
m = Message(int(post.attrib['id']),
post.attrib['time'],
post.find('login').text,
post.find('message').text,
post.find('login').text.lower() == self.browser.username.lower())
if last is not None and last == m.id:
break
msgs.append(m)
return msgs
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dlfp/pages/index.py 0000664 0000000 0000000 00000002502 11666415431 0027403 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage
class DLFPPage(BasePage):
def is_logged(self):
for form in self.document.getiterator('form'):
if form.attrib.get('id', None) == 'new_account_sidebar':
return False
return True
class IndexPage(DLFPPage):
def get_login_token(self):
form = self.parser.select(self.document.getroot(), 'form#new_account_sidebar', 1)
for i in form.find('div').getiterator('input'):
if i.attrib['name'] == 'authenticity_token':
return i.attrib['value']
class LoginPage(DLFPPage):
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dlfp/pages/news.py 0000664 0000000 0000000 00000016527 11666415431 0027264 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import datetime
from weboob.tools.browser import BrokenPageError
from weboob.tools.misc import local2utc
from weboob.backends.dlfp.tools import url2id
from .index import DLFPPage
class Content(object):
TAGGABLE = False
def __init__(self, browser):
self.browser = browser
self.url = u''
self.id = u''
self.title = u''
self.author = u''
self.username = u''
self.body = u''
self.date = None
self.score = 0
self.comments = []
self.relevance_url = None
self.relevance_token = None
def is_taggable(self):
return False
class Comment(Content):
def __init__(self, article, div, reply_id):
Content.__init__(self, article.browser)
self.reply_id = reply_id
self.signature = u''
self.id = div.attrib['id'].split('-')[1]
self.url = '%s#%s' % (article.url, div.attrib['id'])
self.title = unicode(self.browser.parser.select(div.find('h2'), 'a.title', 1).text)
try:
a = self.browser.parser.select(div.find('p'), 'a[rel=author]', 1)
except BrokenPageError:
self.author = 'Anonyme'
self.username = None
else:
self.author = unicode(a.text)
self.username = unicode(a.attrib['href'].split('/')[2])
self.date = datetime.strptime(self.browser.parser.select(div.find('p'), 'time', 1).attrib['datetime'].split('+')[0],
'%Y-%m-%dT%H:%M:%S')
self.date = local2utc(self.date)
content = div.find('div')
try:
signature = self.browser.parser.select(content, 'p.signature', 1)
except BrokenPageError:
# No signature.
pass
else:
content.remove(signature)
self.signature = self.browser.parser.tostring(signature)
self.body = self.browser.parser.tostring(content)
self.score = int(self.browser.parser.select(div.find('p'), 'span.score', 1).text)
forms = self.browser.parser.select(div.find('footer'), 'form.button_to')
if len(forms) > 0:
self.relevance_url = forms[0].attrib['action'].rstrip('for').rstrip('against')
self.relevance_token = self.browser.parser.select(forms[0], 'input[name=authenticity_token]', 1).attrib['value']
subs = div.find('ul')
if subs is not None:
for sub in subs.findall('li'):
comment = Comment(article, sub, self.id)
self.comments.append(comment)
def iter_all_comments(self):
for comment in self.comments:
yield comment
for c in comment.iter_all_comments():
yield c
def __repr__(self):
return u"" % (self.id, self.author, self.title)
class Article(Content):
TAGGABLE = True
def __init__(self, browser, url, tree):
Content.__init__(self, browser)
self.url = url
self.id = url2id(self.url)
if tree is None:
return
header = tree.find('header')
self.title = u' — '.join([a.text for a in header.find('h1').findall('a')])
try:
a = self.browser.parser.select(header, 'a[rel=author]', 1)
except BrokenPageError:
self.author = 'Anonyme'
self.username = None
else:
self.author = unicode(a.text)
self.username = unicode(a.attrib['href'].split('/')[2])
self.body = self.browser.parser.tostring(self.browser.parser.select(tree, 'div.content', 1))
try:
self.date = datetime.strptime(self.browser.parser.select(header, 'time', 1).attrib['datetime'].split('+')[0],
'%Y-%m-%dT%H:%M:%S')
self.date = local2utc(self.date)
except BrokenPageError:
pass
for form in self.browser.parser.select(tree.find('footer'), 'form.button_to'):
if form.attrib['action'].endswith('/for'):
self.relevance_url = form.attrib['action'].rstrip('for').rstrip('against')
self.relevance_token = self.browser.parser.select(form, 'input[name=authenticity_token]', 1).attrib['value']
self.score = int(self.browser.parser.select(tree, 'div.figures figure.score', 1).text)
def append_comment(self, comment):
self.comments.append(comment)
def iter_all_comments(self):
for comment in self.comments:
yield comment
for c in comment.iter_all_comments():
yield c
class CommentPage(DLFPPage):
def get_comment(self):
article = Article(self.browser, self.url, None)
return Comment(article, self.parser.select(self.document.getroot(), 'li.comment', 1), 0)
class ContentPage(DLFPPage):
def on_loaded(self):
self.article = None
def is_taggable(self):
return True
def get_comment(self, id):
article = Article(self.browser, self.url, None)
try:
li = self.parser.select(self.document.getroot(), 'li#comment-%s' % id, 1)
except BrokenPageError:
return None
else:
return Comment(article, li, 0)
def get_article(self):
if not self.article:
self.article = Article(self.browser,
self.url,
self.parser.select(self.document.getroot(), 'div#contents article', 1))
try:
threads = self.parser.select(self.document.getroot(), 'ul.threads', 1)
except BrokenPageError:
pass # no comments
else:
for comment in threads.findall('li'):
self.article.append_comment(Comment(self.article, comment, 0))
return self.article
def get_post_comment_url(self):
return self.parser.select(self.document.getroot(), 'p#send-comment', 1).find('a').attrib['href']
def get_tag_url(self):
return self.parser.select(self.document.getroot(), 'div.tag_in_place', 1).find('a').attrib['href']
class NewCommentPage(DLFPPage):
pass
class NewTagPage(DLFPPage):
def _is_tag_form(self, form):
return form.action.endswith('/tags')
def tag(self, tag):
self.browser.select_form(predicate=self._is_tag_form)
self.browser['tags'] = tag
self.browser.submit()
class NodePage(DLFPPage):
def get_errors(self):
try:
div = self.parser.select(self.document.getroot(), 'div.errors', 1)
except BrokenPageError:
return []
l = []
for li in div.find('ul').findall('li'):
l.append(li.text)
return l
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dlfp/pages/wiki.py 0000664 0000000 0000000 00000004043 11666415431 0027241 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BrokenPageError
from .index import DLFPPage
class WikiEditPage(DLFPPage):
def get_body(self):
try:
return self.parser.select(self.document.getroot(), 'textarea#wiki_page_wiki_body', 1).text
except BrokenPageError:
return ''
def _is_wiki_form(self, form):
return form.attrs.get('class', '') in ('new_wiki_page', 'edit_wiki_page')
def post_content(self, title, body, message):
self.browser.select_form(predicate=self._is_wiki_form)
self.browser.set_all_readonly(False)
if title is not None:
self.browser['wiki_page[title]'] = title.encode('utf-8')
self.browser['commit'] = 'Créer'
else:
self.browser['commit'] = 'Mettre à jour'
self.browser['wiki_page[wiki_body]'] = body.encode('utf-8')
if message is not None:
self.browser['wiki_page[message]'] = message.encode('utf-8')
self.browser.submit()
def post_preview(self, body):
self.browser.select_form(predicate=self._is_wiki_form)
self.browser['wiki_page[wiki_body]'] = body
self.browser.submit()
def get_preview_html(self):
body = self.parser.select(self.document.getroot(), 'article.wikipage div.content', 1)
return self.parser.tostring(body)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dlfp/test.py 0000664 0000000 0000000 00000003460 11666415431 0026160 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import datetime
from weboob.tools.test import BackendTest
from weboob.backends.dlfp.browser import DLFP
__all__ = ['DLFPTest']
class DLFPTest(BackendTest):
BACKEND = 'dlfp'
def __init__(self, *args, **kwargs):
DLFP.DOMAIN = 'alpha.linuxfr.org'
BackendTest.__init__(self, *args, **kwargs)
def test_new_messages(self):
feeds = {}
for name, feed in self.backend.FEEDS.iteritems():
feeds[name] = feed.replace('//linuxfr.org', '//alpha.linuxfr.org')
self.backend.FEEDS = feeds
for message in self.backend.iter_unread_messages():
pass
def test_get_content(self):
self.backend.get_content(u"Ceci-est-un-test")
def test_push_content(self):
content = self.backend.get_content(u"Ceci-est-un-test")
content.content = "test "+str(datetime.now())
self.backend.push_content(content, message="test weboob", minor=True)
def test_content_preview(self):
content = self.backend.get_content(u"Ceci-est-un-test")
self.backend.get_content_preview(content)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/dlfp/tools.py 0000664 0000000 0000000 00000004322 11666415431 0026337 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
RSSID_RE = re.compile('tag:.*:(\w+)/(\d+)')
ID2URL_RE = re.compile('^(\w)(.*)\.([^ \.]+)$')
REGEXPS = {'/users/%s/journaux/%s': 'D%s.%s',
'/news/%s': 'N.%s',
'/wiki/%s': 'W.%s',
'/suivi/%s': 'T.%s',
'/sondages/%s': 'P.%s',
'/forums/%s/posts/%s': 'B%s.%s',
}
def f2re(f):
return '.*' + f.replace('%s', '([^ /]+)')
def rssid(entry):
m = RSSID_RE.match(entry.id)
if not m:
return None
ind = m.group(1).replace('Post', 'Board')[0]
for url_re, id_re in REGEXPS.iteritems():
if id_re[0] != ind:
continue
if id_re.count('%s') == 2:
mm = re.match(f2re(url_re), entry.link)
if not mm:
return
return '%s%s.%s' % (ind, mm.group(1), m.group(2))
else:
return '%s.%s' % (ind, m.group(2))
def id2url(id):
m = ID2URL_RE.match(id)
if not m:
return None
for url_re, id_re in REGEXPS.iteritems():
if id_re[0] != m.group(1):
continue
if id_re.count('%s') == 2:
return url_re % (m.group(2), m.group(3))
else:
return url_re % m.group(3)
def url2id(url):
for url_re, id_re in REGEXPS.iteritems():
m = re.match(f2re(url_re), url)
if not m:
continue
return id_re % m.groups()
def id2threadid(id):
m = ID2URL_RE.match(id)
if m:
return m.group(3)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/eatmanga/ 0000775 0000000 0000000 00000000000 11666415431 0025454 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/eatmanga/__init__.py 0000664 0000000 0000000 00000001440 11666415431 0027564 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import EatmangaBackend
__all__ = ['EatmangaBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/eatmanga/backend.py 0000664 0000000 0000000 00000002571 11666415431 0027422 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderBackend, DisplayPage
__all__ = ['EatmangaBackend']
class EatmangaBackend(GenericComicReaderBackend):
NAME = 'eatmanga'
DESCRIPTION = 'Eatmanga manga reading site'
DOMAIN = 'www.eatmanga.com'
BROWSER_PARAMS = dict(
img_src_xpath="//img[@class='eatmanga_bigimage']/@src",
page_list_xpath="(//select[@id='pages'])[1]/option/@value")
ID_REGEXP = r'[^/]+/[^/]+'
URL_REGEXP = r'.+eatmanga.com/(?:index.php/)?Manga-Scan/(%s).+' % ID_REGEXP
ID_TO_URL = 'http://www.eatmanga.com/index.php/Manga-Scan/%s'
PAGES = { URL_REGEXP: DisplayPage }
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/eatmanga/test.py 0000664 0000000 0000000 00000001734 11666415431 0027012 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderTest
class EatmangaTest(GenericComicReaderTest):
BACKEND = 'eatmanga'
def test_download(self):
return self._test_download('Glass-Mask/Glass-Mask-Vol-031')
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ecrans/ 0000775 0000000 0000000 00000000000 11666415431 0025152 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ecrans/__init__.py 0000664 0000000 0000000 00000001514 11666415431 0027264 0 ustar 00root root 0000000 0000000 "NewspaperEcransBackend init"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import NewspaperEcransBackend
__all__ = ['NewspaperEcransBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ecrans/backend.py 0000664 0000000 0000000 00000002566 11666415431 0027124 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
"backend for http://www.ecrans.fr"
from weboob.capabilities.messages import ICapMessages
from weboob.tools.capabilities.messages.GenericBackend import GenericNewspaperBackend
from .browser import NewspaperEcransBrowser
from .tools import rssid
class NewspaperEcransBackend(GenericNewspaperBackend, ICapMessages):
"NewspaperEcransBackend class"
MAINTAINER = 'Julien Hebert'
EMAIL = 'juke@free.fr'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
STORAGE = {'seen': {}}
NAME = 'ecrans'
DESCRIPTION = u'Ecrans French news website'
BROWSER = NewspaperEcransBrowser
RSS_FEED = 'http://www.ecrans.fr/spip.php?page=backend'
RSSID = rssid
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ecrans/browser.py 0000664 0000000 0000000 00000002376 11666415431 0027217 0 ustar 00root root 0000000 0000000 "browser for ecrans website"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .pages.article import ArticlePage
from weboob.tools.browser import BaseBrowser
class NewspaperEcransBrowser(BaseBrowser):
"NewspaperEcransBrowser class"
PAGES = {
"http://www.ecrans.fr/.*": ArticlePage,
}
def is_logged(self):
return False
def login(self):
pass
def fillobj(self, obj, fields):
pass
def get_content(self, _id):
"return page article content"
self.location(_id)
return self.page.get_article(_id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ecrans/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026251 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ecrans/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0030350 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ecrans/pages/article.py 0000664 0000000 0000000 00000003117 11666415431 0030250 0 ustar 00root root 0000000 0000000 "ArticlePage object for inrocks"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage, remove_from_selector_list, try_remove_from_selector_list, try_drop_tree
class ArticlePage(GenericNewsPage):
"ArticlePage object for inrocks"
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_title_selector = "title"
self.element_author_selector = "p.auteur>a"
self.element_body_selector = "div.bloc_article_01"
def get_body(self):
element_body = self.get_element_body()
remove_from_selector_list(self.parser, element_body, ["p.auteur", "h4" ])
try_remove_from_selector_list(self.parser, element_body, ["p.tag", "div.alire", self.element_title_selector, "h4"])
try_drop_tree(self.parser, element_body, "script")
return self.parser.tostring(element_body)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ecrans/test.py 0000664 0000000 0000000 00000001715 11666415431 0026507 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
__all__ = ['EcransTest']
class EcransTest(BackendTest):
BACKEND = 'ecrans'
def test_new_messages(self):
for message in self.backend.iter_unread_messages():
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ecrans/tools.py 0000664 0000000 0000000 00000002417 11666415431 0026670 0 ustar 00root root 0000000 0000000 "tools for lefigaro backend"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
def id2url(_id):
"return an url from an id"
regexp2 = re.compile("(\w+).([0-9]+).(.*$)")
match = regexp2.match(_id)
if match:
return 'http://www.20minutes.fr/%s/%s/%s' % ( match.group(1),
match.group(2),
match.group(3))
else:
raise ValueError("id doesn't match")
def url2id(url):
"return an id from an url"
return url
def rssid(entry):
return url2id(entry.id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ehentai/ 0000775 0000000 0000000 00000000000 11666415431 0025314 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ehentai/__init__.py 0000664 0000000 0000000 00000001443 11666415431 0027427 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import EHentaiBackend
__all__ = ['EHentaiBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ehentai/backend.py 0000664 0000000 0000000 00000006442 11666415431 0027263 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
import re
from weboob.capabilities.gallery import ICapGallery
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.misc import ratelimit
from weboob.tools.value import Value, ValueBackendPassword
from .browser import EHentaiBrowser
from .gallery import EHentaiGallery, EHentaiImage
__all__ = ['EHentaiBackend']
class EHentaiBackend(BaseBackend, ICapGallery):
NAME = 'ehentai'
MAINTAINER = 'Roger Philibert'
EMAIL = 'roger.philibert@gmail.com'
VERSION = '0.9.1'
DESCRIPTION = 'E-hentai galleries'
LICENSE = 'AGPLv3+'
BROWSER = EHentaiBrowser
CONFIG = BackendConfig(
Value('domain', label='Domain', default='g.e-hentai.org'),
Value('username', label='Username', default=''),
ValueBackendPassword('password', label='Password'))
def create_default_browser(self):
username = self.config['username'].get()
if username:
password = self.config['password'].get()
else:
password = None
return self.create_browser(self.config['domain'].get(), username, password)
def iter_search_results(self, pattern=None, sortby=None, max_results=None):
with self.browser:
return self.browser.iter_search_results(pattern)
def iter_gallery_images(self, gallery):
self.fillobj(gallery, ('url',))
with self.browser:
return self.browser.iter_gallery_images(gallery)
ID_REGEXP = r'/?\d+/[\dabcdef]+/?'
URL_REGEXP = r'.+/g/(%s)' % ID_REGEXP
def get_gallery(self, _id):
match = re.match(r'^%s$' % self.URL_REGEXP, _id)
if match:
_id = match.group(1)
else:
match = re.match(r'^%s$' % self.ID_REGEXP, _id)
if match:
_id = match.group(0)
else:
return None
gallery = EHentaiGallery(_id)
with self.browser:
if self.browser.gallery_exists(gallery):
return gallery
else:
return None
def fill_gallery(self, gallery, fields):
if not gallery.__iscomplete__():
with self.browser:
self.browser.fill_gallery(gallery, fields)
def fill_image(self, image, fields):
with self.browser:
image.url = self.browser.get_image_url(image)
if 'data' in fields:
ratelimit("ehentai_get", 2)
image.data = self.browser.readurl(image.url)
OBJECTS = {
EHentaiGallery: fill_gallery,
EHentaiImage: fill_image }
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ehentai/browser.py 0000664 0000000 0000000 00000006773 11666415431 0027366 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from urllib import urlencode
from .pages import IndexPage, GalleryPage, ImagePage, HomePage, LoginPage
from .gallery import EHentaiImage
__all__ = ['EHentaiBrowser']
class EHentaiBrowser(BaseBrowser):
ENCODING = None
PAGES = {
r'http://[^/]+/': IndexPage,
r'http://[^/]+/\?.*': IndexPage,
r'http://[^/]+/g/.+': GalleryPage,
r'http://[^/]+/s/.*': ImagePage,
r'http://[^/]+/home\.php': HomePage,
r'http://e-hentai\.org/bounce_login\.php': LoginPage,
}
def __init__(self, domain, username, password, *args, **kwargs):
self.DOMAIN = domain
self.logged = False
BaseBrowser.__init__(self, parser=('lxmlsoup',), *args, **kwargs)
if password:
self.login(username, password)
def _gallery_url(self, gallery):
return 'http://%s/g/%s/' % (self.DOMAIN, gallery.id)
def _gallery_page(self, gallery, n):
return gallery.url + ('?p=%d' % n)
def iter_search_results(self, pattern):
self.location(self.buildurl('/', f_search=pattern.encode('utf-8')))
assert self.is_on_page(IndexPage)
return self.page.iter_galleries()
def iter_gallery_images(self, gallery):
self.location(gallery.url)
assert self.is_on_page(GalleryPage)
i = 0
while True:
n = self.page._next_page_link();
for img in self.page.image_pages():
yield EHentaiImage(img)
if n is None:
break
i += 1
self.location(self._gallery_page(gallery, i))
assert self.is_on_page(GalleryPage)
def get_image_url(self, image):
self.location(image.id)
assert self.is_on_page(ImagePage)
return self.page.get_url()
def gallery_exists(self, gallery):
gallery.url = self._gallery_url(gallery)
self.location(gallery.url)
assert self.is_on_page(GalleryPage)
return self.page.gallery_exists(gallery)
def fill_gallery(self, gallery, fields):
gallery.url = self._gallery_url(gallery)
self.location(gallery.url)
assert self.is_on_page(GalleryPage)
self.page.fill_gallery(gallery)
def login(self, username, password):
assert isinstance(username, basestring)
assert isinstance(password, basestring)
data = {'ipb_login_username': username,
'ipb_login_password': password}
self.location('http://e-hentai.org/bounce_login.php', urlencode(data), no_login=True)
assert self.is_on_page(LoginPage)
if not self.page.is_logged():
raise BrowserIncorrectPassword()
# necessary in order to reach the fjords
self.home()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ehentai/gallery.py 0000664 0000000 0000000 00000002142 11666415431 0027324 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.gallery import BaseGallery, BaseImage
__all_ = ['EHentaiGallery', 'EHentaiImage']
class EHentaiGallery(BaseGallery):
def __init__(self, *args, **kwargs):
BaseGallery.__init__(self, *args, **kwargs)
self.nsfw = True
class EHentaiImage(BaseImage):
def __init__(self, *args, **kwargs):
BaseImage.__init__(self, *args, **kwargs)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ehentai/pages.py 0000664 0000000 0000000 00000010112 11666415431 0026760 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage
from weboob.tools.misc import html2text
from weboob.tools.capabilities.thumbnail import Thumbnail
from datetime import datetime
import re
from .gallery import EHentaiGallery
__all__ = ['GalleryPage', 'ImagePage', 'IndexPage', 'HomePage', 'LoginPage']
class LoginPage(BasePage):
def is_logged(self):
success_p = self.document.xpath(
'//p[text() = "Login Successful. You will be returned momentarily."]')
if len(success_p):
print 'logged on'
return True
else:
print 'not logged on'
return False
class HomePage(BasePage):
pass
class IndexPage(BasePage):
def iter_galleries(self):
lines = self.document.xpath('//table[@class="itg"]//tr[@class="gtr0" or @class="gtr1"]')
for line in lines:
a = line.xpath('.//div[@class="it3"]/a')[-1]
url = a.attrib["href"]
title = a.text.strip()
yield EHentaiGallery(re.search('(?<=/g/)\d+/[\dabcdef]+', url).group(0), title=title)
class GalleryPage(BasePage):
def image_pages(self):
return self.document.xpath('//div[@class="gdtm"]//a/attribute::href')
def _next_page_link(self):
try:
return self.document.xpath("//table[@class='ptt']//a[text()='>']")[0]
except IndexError:
return None
def gallery_exists(self, gallery):
if self.document.xpath("//h1"):
return True
else:
return False
def fill_gallery(self, gallery):
gallery.title = self.document.xpath("//h1[@id='gn']/text()")[0]
try:
gallery.original_title = self.document.xpath("//h1[@id='gj']/text()")[0]
except IndexError:
gallery.orginal_title = None
description_div = self.document.xpath("//div[@id='gds']/div")[0]
description_html = self.parser.tostring(description_div)
gallery.description = html2text(description_html)
cardinality_string = self.document.xpath("//div[@id='gdd']//tr[td[@class='gdt1']/text()='Images:']/td[@class='gdt2']/text()")[0]
gallery.cardinality = int(re.match(r"\d+", cardinality_string).group(0))
date_string = self.document.xpath("//div[@id='gdd']//tr[td[@class='gdt1']/text()='Posted:']/td[@class='gdt2']/text()")[0]
gallery.date = datetime.strptime(date_string, "%Y-%m-%d %H:%M")
rating_string = self.document.xpath("//td[@id='rating_label']/text()")[0]
rating_match = re.search(r"\d+\.\d+", rating_string)
if rating_match is None:
gallery.rating = None
else:
gallery.rating = float(rating_match.group(0))
gallery.rating_max = 5
try:
thumbnail_url = self.document.xpath("//div[@class='gdtm']/a/img/attribute::src")[0]
except IndexError:
thumbnail_style = self.document.xpath("//div[@class='gdtm']/div/attribute::style")[0]
thumbnail_url = re.search(r"background:[^;]+url\((.+?)\)", thumbnail_style).group(1)
gallery.thumbnail = Thumbnail(thumbnail_url)
def _prev_page_link(self):
try:
return self.document.xpath("//table[@class='ptt']//a[text()='<']")[0]
except IndexError:
return None
class ImagePage(BasePage):
def get_url(self):
return self.document.xpath('//div[@class="sni"]/a/img/attribute::src')[0]
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ehentai/test.py 0000664 0000000 0000000 00000002667 11666415431 0026660 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class EHentaiTest(BackendTest):
BACKEND = 'ehentai'
def test_ehentai(self):
l = list(self.backend.iter_search_results('lol'))
self.assertTrue(len(l) > 0)
v = l[0]
self.backend.fillobj(v, ('url',))
self.assertTrue(v.url and v.url.startswith('http://'), 'URL for gallery "%s" not found: %s' % (v.id, v.url))
self.backend.browser.openurl(v.url)
img = self.backend.iter_gallery_images(v).next()
self.backend.fillobj(img, ('url',))
self.assertTrue(v.url and v.url.startswith('http://'), 'URL for first image in gallery "%s" not found: %s' % (v.id, img.url))
self.backend.browser.openurl(img.url)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/fourchan/ 0000775 0000000 0000000 00000000000 11666415431 0025504 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/fourchan/__init__.py 0000664 0000000 0000000 00000000156 11666415431 0027617 0 ustar 00root root 0000000 0000000 from .backend import FourChanBackend
from .browser import FourChan
__all__ = ['FourChanBackend', 'FourChan']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/fourchan/backend.py 0000664 0000000 0000000 00000010266 11666415431 0027452 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.messages import ICapMessages, Message, Thread
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import Value
from .browser import FourChan
__all__ = ['FourChanBackend']
class FourChanBackend(BaseBackend, ICapMessages):
NAME = 'fourchan'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = '4chan website'
CONFIG = BackendConfig(Value('boards', label='Boards to fetch'))
STORAGE = {'boards': {}}
BROWSER = FourChan
def _splitid(self, id):
return id.split('.', 1)
def get_thread(self, id):
thread = None
if isinstance(id, Thread):
thread = id
id = thread.id
if not '.' in id:
self.logger.warning('Malformated ID (%s)' % id)
return
board, thread_id = self._splitid(id)
with self.browser:
_thread = self.browser.get_thread(board, thread_id)
flags = 0
if not _thread.id in self.storage.get('boards', board, default={}):
flags |= Message.IS_UNREAD
if not thread:
thread = Thread(id)
thread.title = _thread.filename
thread.root = Message(thread=thread,
id=0, # root message
title=_thread.filename,
sender=_thread.author,
receivers=None,
date=_thread.datetime,
parent=None,
content=_thread.text,
signature=None,
children=[],
flags=flags|Message.IS_HTML)
for comment in _thread.comments:
flags = 0
if not comment.id in self.storage.get('boards', board, _thread.id, default=[]):
flags |= Message.IS_UNREAD
m = Message(thread=thread,
id=comment.id,
title=_thread.filename,
sender=comment.author,
receivers=None,
date=comment.datetime,
parent=thread.root,
content=comment.text,
signature=None,
children=None,
flags=flags|Message.IS_HTML)
thread.root.children.append(m)
return thread
def iter_threads(self):
for board in self.config['boards'].get().split(' '):
with self.browser:
threads = self.browser.get_threads(board)
for thread in threads:
t = Thread('%s.%s' % (board, thread.id))
t.title = thread.filename
yield t
def iter_unread_messages(self):
for thread in self.iter_threads():
self.fill_thread(thread, 'root')
for m in thread.iter_all_messages():
if m.flags & Message.IS_UNREAD:
yield m
def set_message_read(self, message):
board, thread_id = self._splitid(message.thread.id)
self.storage.set('boards', board, thread_id, self.storage.get('boards', board, thread_id, default=[]) + [message.id])
self.storage.save()
def fill_thread(self, thread, fields):
return self.get_thread(thread)
OBJECTS = {Thread: fill_thread}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/fourchan/browser.py 0000664 0000000 0000000 00000002553 11666415431 0027546 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser
from .pages.board import BoardPage
class FourChan(BaseBrowser):
DOMAIN = 'boards.4chan.org'
PAGES = {
'http://boards.4chan.org/\w+/': BoardPage,
'http://boards.4chan.org/\w+/res/\d+': BoardPage,
}
def is_logged(self):
return True
def get_threads(self, board):
self.location('http://boards.4chan.org/%s/' % board)
return self.page.articles
def get_thread(self, board, id):
self.location('http://boards.4chan.org/%s/res/%d' % (board, long(id)))
assert len(self.page.articles) == 1
return self.page.articles[0]
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/fourchan/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026603 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/fourchan/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0030702 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/fourchan/pages/board.py 0000664 0000000 0000000 00000006232 11666415431 0030247 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from datetime import datetime
from weboob.tools.browser import BasePage
__all__ = ['BoardPage']
class Message(object):
def __init__(self, browser, board, id, filename=u'', url=u''):
self.id = id
self.browser = browser
self.board = board
self.filename = filename
self.datetime = datetime.now()
self.url = url
self.author = u''
self.text = u''
self.comments = []
def add_comment(self, div):
comment = Message(self.browser, self.board, int(div.attrib.get('id', '')))
comment.author = div.cssselect('span.commentpostername')[0].text
comment.text = self.browser.parser.tostring(div.find('blockquote'))
self.comments.append(comment)
def __repr__(self):
return '' % (self.id, self.filename, self.url, len(self.comments))
class BoardPage(BasePage):
URL_REGEXP = re.compile('http://boards.4chan.org/(\w+)/')
def on_loaded(self):
self.articles = []
m = self.URL_REGEXP.match(self.url)
if m:
self.board = m.group(1)
else:
self.logger.warning('Unable to find board')
self.board = 'unknown'
forms = self.document.getroot().cssselect('form')
form = None
for f in forms:
if f.attrib.get('name', '') == 'delform':
form = f
break
if form is None:
self.logger.warning('No delform :(')
article = None
for div in form.getchildren():
if div.tag == 'span' and div.attrib.get('class', '') == 'filesize':
url = div.find('a').get('href', '')
filename = 'unknown.jpg'
span = div.find('span')
if span is not None:
filename = span.text
article = Message(self.browser, self.board, 0, filename, url)
self.articles.append(article)
if article is None:
continue
if div.tag == 'input' and div.attrib.get('type', 'checkbox') and div.attrib.get('value', 'delete'):
article.id = int(div.attrib.get('name', '0'))
if div.tag == 'blockquote':
article.text = self.parser.tostring(div)
if div.tag == 'table':
tags = div.cssselect('td.reply')
if tags:
article.add_comment(tags[0])
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/fourchan/test.py 0000664 0000000 0000000 00000002537 11666415431 0027044 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from logging import debug
from weboob.tools.test import BackendTest
class FourChanTest(BackendTest):
BACKEND = 'fourchan'
def test_new_messages(self):
tot = 0
for thread in self.backend.iter_threads():
thread = self.backend.fillobj(thread, 'root')
count = 0
for m in thread.iter_all_messages():
count += 1
debug('Count: %s' % count)
tot += count
debug('Total messages: %s' % tot)
count = 0
for message in self.backend.iter_unread_messages():
count += 1
debug('Unread messages: %s' % count)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/franceinter/ 0000775 0000000 0000000 00000000000 11666415431 0026177 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/franceinter/__init__.py 0000664 0000000 0000000 00000001453 11666415431 0030313 0 ustar 00root root 0000000 0000000 # * -*- coding: utf-8 -*-
# Copyright(C) 2011 Johann Broudin
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import FranceInterBackend
__all__ = ['FranceInterBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/franceinter/backend.py 0000664 0000000 0000000 00000006520 11666415431 0030143 0 ustar 00root root 0000000 0000000 # * -*- coding: utf-8 -*-
# Copyright(C) 2011 Johann Broudin
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.radio import ICapRadio, Radio, Stream, Emission
from weboob.capabilities.collection import ICapCollection, CollectionNotFound
from weboob.tools.backend import BaseBackend
from weboob.tools.browser import BaseBrowser, BasePage
__all__ = ['FranceInterBackend']
class XMLinfos(BasePage):
def get_current(self):
emissions = self.parser.select(self.document.getroot(), 'item')
if len(emissions) == 0:
return 'No emission'
return emissions[0].find('titreemission').text
class FranceInterBrowser(BaseBrowser):
DOMAIN = u'metadatas.tv-radio.com'
ENCODING = 'iso-8859-1'
PAGES = {r'.*metadatas/franceinterRSS\.xml': XMLinfos}
def get_current(self, radio):
self.location('/metadatas/franceinterRSS.xml')
assert self.is_on_page(XMLinfos)
return self.page.get_current()
class FranceInterBackend(BaseBackend, ICapRadio, ICapCollection):
NAME = 'franceinter'
MAINTAINER = 'Johann Broudin'
EMAIL = 'johann.broudin@6-8.fr'
VERSION = '0.9.1'
DESCRIPTION = u'The france inter french radio'
LICENCE = 'AGPLv3+'
BROWSER = FranceInterBrowser
_RADIOS = {'franceinter': (u'france inter', u'france inter', u'http://mp3.live.tv-radio.com/franceinter/all/franceinterhautdebit.mp3')}
def iter_resources(self, splited_path):
if len(splited_path) > 0:
raise CollectionNotFound()
for id in self._RADIOS.iterkeys():
yield self.get_radio(id)
def iter_radios_search(self, pattern):
for radio in self.iter_resources([]):
if pattern.lower() in radio.title.lower() or pattern.lower() in radio.description.lower():
yield radio
def get_radio(self, radio):
if not isinstance(radio, Radio):
radio = Radio(radio)
if not radio.id in self._RADIOS:
return None
title, description, url = self._RADIOS[radio.id]
radio.title = title
radio.description = description
emission = self.browser.get_current(radio.id)
current = Emission(0)
current.title = unicode(emission)
current.artist = None
radio.current = current
stream = Stream(0)
stream.title = u'128kbits/s'
stream.url = url
radio.streams = [stream]
return radio
def fill_radio(self, radio, fields):
if 'current' in fields:
if not radio.current:
radio.current = Emission(0)
radio.current.artist = self.browser.get_current(radio.id)
radio.current.title = None
return radio
OBJECTS = {Radio: fill_radio}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/franceinter/test.py 0000664 0000000 0000000 00000001702 11666415431 0027530 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class FranceInterTest(BackendTest):
BACKEND = 'franceinter'
def test_franceinter(self):
l = list(self.backend.iter_resources([]))
self.assertTrue(len(l) > 0)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/francetelevisions/ 0000775 0000000 0000000 00000000000 11666415431 0027422 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/francetelevisions/__init__.py 0000664 0000000 0000000 00000000076 11666415431 0031536 0 ustar 00root root 0000000 0000000 from .backend import PluzzBackend
__all__ = ['PluzzBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/francetelevisions/backend.py 0000664 0000000 0000000 00000003770 11666415431 0031372 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.video import ICapVideo
from weboob.tools.backend import BaseBackend
from .browser import PluzzBrowser
from .video import PluzzVideo
__all__ = ['PluzzBackend']
class PluzzBackend(BaseBackend, ICapVideo):
NAME = 'francetelevisions'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
DESCRIPTION = 'France Televisions video website'
LICENSE = 'AGPLv3+'
BROWSER = PluzzBrowser
def get_video(self, _id):
with self.browser:
return self.browser.get_video(_id)
def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
with self.browser:
return self.browser.iter_search_results(pattern)
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
with self.browser:
video = self.browser.get_video(PluzzVideo.id2url(video.id), video)
if 'thumbnail' in fields and video.thumbnail:
with self.browser:
video.thumbnail.data = self.browser.readurl(video.thumbnail.url)
return video
OBJECTS = {PluzzVideo: fill_video}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/francetelevisions/browser.py 0000664 0000000 0000000 00000003770 11666415431 0031466 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser
from weboob.tools.browser.decorators import id2url
from .pages import IndexPage, VideoPage, MetaVideoPage
from .video import PluzzVideo
__all__ = ['PluzzBrowser']
class PluzzBrowser(BaseBrowser):
DOMAIN = 'pluzz.fr'
ENCODING = None
PAGES = {r'http://[w\.]*pluzz.fr/?': IndexPage,
r'http://[w\.]*pluzz.fr/recherche.html.*': IndexPage,
r'http://[w\.]*pluzz.fr/[-\w]+/.*': IndexPage,
r'http://[w\.]*pluzz.fr/((?!recherche).+)\.html': VideoPage,
r'http://info\.francetelevisions\.fr/\?id-video=.*': MetaVideoPage,
}
@id2url(PluzzVideo.id2url)
def get_video(self, url, video=None):
self.location(url)
assert self.is_on_page(VideoPage)
id = self.page.get_id()
metaurl = self.page.get_meta_url()
if metaurl is None:
return None
self.location(metaurl)
assert self.is_on_page(MetaVideoPage)
return self.page.get_video(id, video)
def iter_search_results(self, pattern):
if not pattern:
self.home()
else:
self.location(self.buildurl('recherche.html', q=pattern.encode('utf-8')))
assert self.is_on_page(IndexPage)
return self.page.iter_videos()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/francetelevisions/pages.py 0000664 0000000 0000000 00000005765 11666415431 0031110 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import datetime
import re
from weboob.tools.capabilities.thumbnail import Thumbnail
from weboob.tools.browser import BasePage, BrokenPageError
from .video import PluzzVideo
__all__ = ['IndexPage', 'VideoPage']
class IndexPage(BasePage):
def iter_videos(self):
for div in self.parser.select(self.document.getroot(), 'li.vignette'):
url = self.parser.select(div, 'h4 a', 1).attrib['href']
m = re.match('http://www.pluzz.fr/([^/]+).html', url)
if not m:
print ':('
continue
_id = m.group(1)
video = PluzzVideo(_id)
video.title = self.parser.select(div, 'h4 a', 1).text
m = re.match('(\d+)/(\d+)/(\d+)', self.parser.select(div, 'p.date', 1).text)
if m:
video.date = datetime.datetime(int(m.group(3)),
int(m.group(2)),
int(m.group(1)))
url = self.parser.select(div, 'img.illustration', 1).attrib['src']
video.thumbnail = Thumbnail('http://www.pluzz.fr/%s' % url)
yield video
class VideoPage(BasePage):
def on_loaded(self):
p = self.parser.select(self.document.getroot(), 'p.alert')
if len(p) > 0:
raise Exception(p[0].text)
def get_meta_url(self):
try:
div = self.parser.select(self.document.getroot(), 'a#current_video', 1)
except BrokenPageError:
return None
else:
return div.attrib['href']
def get_id(self):
return self.groups[0]
class MetaVideoPage(BasePage):
def get_meta(self, name):
return self.parser.select(self.document.getroot(), 'meta[name=%s]' % name, 1).attrib['content']
def get_video(self, id, video=None):
if video is None:
video = PluzzVideo(id)
video.title = self.get_meta('vignette-titre-court')
video.url = 'mms://videozones.francetv.fr/%s' % self.get_meta('urls-url-video')[len('geoloc/'):]
video.description = self.get_meta('description')
hours, minutes, seconds = self.get_meta('vignette-duree').split(':')
video.duration = datetime.timedelta(hours=int(hours), minutes=int(minutes), seconds=int(seconds))
return video
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/francetelevisions/test.py 0000664 0000000 0000000 00000002170 11666415431 0030753 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class PluzzTest(BackendTest):
BACKEND = 'francetelevisions'
def test_francetelevisions(self):
l = list(self.backend.iter_search_results('jt'))
self.assertTrue(len(l) > 0)
v = l[0]
self.backend.fillobj(v, ('url',))
self.assertTrue(v.url and v.url.startswith('mms://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/francetelevisions/video.py 0000664 0000000 0000000 00000002031 11666415431 0031076 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.video import BaseVideo
__all__ = ['PluzzVideo']
class PluzzVideo(BaseVideo):
def __init__(self, *args, **kwargs):
BaseVideo.__init__(self, *args, **kwargs)
self.ext = 'wmv'
@classmethod
def id2url(cls, _id):
return 'http://www.pluzz.fr/%s.html' % _id
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/gazelle/ 0000775 0000000 0000000 00000000000 11666415431 0025322 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/gazelle/__init__.py 0000664 0000000 0000000 00000000102 11666415431 0027424 0 ustar 00root root 0000000 0000000 from .backend import GazelleBackend
__all__ = ['GazelleBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/gazelle/backend.py 0000664 0000000 0000000 00000004161 11666415431 0027265 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.torrent import ICapTorrent
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import ValueBackendPassword, Value
from .browser import GazelleBrowser
__all__ = ['GazelleBackend']
class GazelleBackend(BaseBackend, ICapTorrent):
NAME = 'gazelle'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
DESCRIPTION = 'gazelle bittorrent tracker'
LICENSE = 'AGPLv3+'
CONFIG = BackendConfig(Value('domain', label='Domain (example "ssl.what.cd")'),
Value('protocol', label='Protocol to use', choices=('http', 'https')),
Value('username', label='Username'),
ValueBackendPassword('password', label='Password'))
BROWSER = GazelleBrowser
def create_default_browser(self):
return self.create_browser(self.config['protocol'].get(), self.config['domain'].get(),
self.config['username'].get(), self.config['password'].get())
def get_torrent(self, id):
return self.browser.get_torrent(id)
def get_torrent_file(self, id):
torrent = self.browser.get_torrent(id)
if not torrent:
return None
return self.browser.openurl(torrent.url.encode('utf-8')).read()
def iter_torrents(self, pattern):
return self.browser.iter_torrents(pattern)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/gazelle/browser.py 0000664 0000000 0000000 00000004370 11666415431 0027363 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser
from .pages.index import IndexPage, LoginPage
from .pages.torrents import TorrentsPage
__all__ = ['GazelleBrowser']
class GazelleBrowser(BaseBrowser):
PAGES = {'https?://[^/]+/?(index.php)?': IndexPage,
'https?://[^/]+/login.php': LoginPage,
'https?://[^/]+/torrents.php.*': TorrentsPage,
}
def __init__(self, protocol, domain, *args, **kwargs):
self.DOMAIN = domain
self.PROTOCOL = protocol
BaseBrowser.__init__(self, *args, **kwargs)
def login(self):
if not self.is_on_page(LoginPage):
self.location('/login.php', no_login=True)
self.page.login(self.username, self.password)
def is_logged(self):
if not self.page or self.is_on_page(LoginPage):
return False
if self.is_on_page(IndexPage):
return self.page.is_logged()
return True
def home(self):
return self.location('%s://%s/' % (self.PROTOCOL, self.DOMAIN))
def iter_torrents(self, pattern):
self.location(self.buildurl('/torrents.php', searchstr=pattern.encode('utf-8')))
assert self.is_on_page(TorrentsPage)
return self.page.iter_torrents()
def get_torrent(self, fullid):
if not '.' in fullid:
return None
id, torrentid = fullid.split('.', 1)
self.location(self.buildurl('/torrents.php', id=id, torrentid=torrentid))
assert self.is_on_page(TorrentsPage)
return self.page.get_torrent(fullid)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/gazelle/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026421 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/gazelle/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0030520 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/gazelle/pages/index.py 0000664 0000000 0000000 00000003140 11666415431 0030100 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage, BrowserIncorrectPassword, BrowserBanned
from weboob.tools.misc import remove_html_tags
__all__ = ['IndexPage', 'LoginPage']
class IndexPage(BasePage):
def is_logged(self):
return 'id' in self.document.find('body').attrib
class LoginPage(BasePage):
def on_loaded(self):
warns = self.parser.select(self.document.getroot(), 'span.warning')
for warn in warns:
text = remove_html_tags(self.parser.tostring(warn)).strip()
if text.startswith('Your username'):
raise BrowserIncorrectPassword(text)
if text.startswith('You are banned'):
raise BrowserBanned(text)
def login(self, login, password):
self.browser.select_form(nr=0)
self.browser['username'] = login
self.browser['password'] = password
self.browser.submit()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/gazelle/pages/torrents.py 0000664 0000000 0000000 00000020342 11666415431 0030654 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
import urlparse
from logging import warning, debug
from weboob.tools.misc import html2text, get_bytes_size
from weboob.tools.browser import BasePage
from weboob.capabilities.torrent import Torrent
from weboob.capabilities.base import NotLoaded
__all__ = ['TorrentsPage']
class TorrentsPage(BasePage):
TORRENTID_REGEXP = re.compile('torrents\.php\?action=download&id=(\d+)')
def format_url(self, url):
return '%s://%s/%s' % (self.browser.PROTOCOL,
self.browser.DOMAIN,
url)
def iter_torrents(self):
table = self.document.getroot().cssselect('table.torrent_table')
if not table:
table = self.document.getroot().cssselect('table#browse_torrent_table')
if table:
table = table[0]
current_group = None
for tr in table.findall('tr'):
if tr.attrib.get('class', '') == 'colhead':
# ignore
continue
if tr.attrib.get('class', '') == 'group':
tds = tr.findall('td')
current_group = u''
div = tds[-6]
if div.getchildren()[0].tag == 'div':
div = div.getchildren()[0]
for a in div.findall('a'):
if not a.text:
continue
if current_group:
current_group += ' - '
current_group += a.text
elif tr.attrib.get('class', '').startswith('group_torrent') or \
tr.attrib.get('class', '').startswith('torrent'):
tds = tr.findall('td')
title = current_group
if len(tds) == 7:
# Under a group
i = 0
elif len(tds) in (8,9):
# An alone torrent
i = len(tds) - 1
while i >= 0 and tds[i].find('a') is None:
i -= 1
else:
# Useless title
continue
if title:
title += u' (%s)' % tds[i].find('a').text
else:
title = tds[i].find('a').text
url = urlparse.urlparse(tds[i].find('a').attrib['href'])
params = urlparse.parse_qs(url.query)
if 'torrentid' in params:
id = '%s.%s' % (params['id'][0], params['torrentid'][0])
else:
url = tds[i].find('span').find('a').attrib['href']
m = self.TORRENTID_REGEXP.match(url)
if not m:
continue
id = '%s.%s' % (params['id'][0], m.group(1))
size, unit = tds[i+3].text.split()
size = get_bytes_size(float(size.replace(',','')), unit)
seeders = int(tds[-2].text)
leechers = int(tds[-1].text)
torrent = Torrent(id,
title,
url=self.format_url(url),
size=size,
seeders=seeders,
leechers=leechers)
yield torrent
else:
debug('unknown attrib: %s' % tr.attrib)
def get_torrent(self, id):
table = self.browser.parser.select(self.document.getroot(), 'div.thin', 1)
h2 = table.find('h2')
if h2 is not None:
title = h2.text or ''
if h2.find('a') != None:
title += (h2.find('a').text or '') + (h2.find('a').tail or '')
else:
title = self.browser.parser.select(table, 'div.title_text', 1).text
torrent = Torrent(id, title)
if '.' in id:
torrentid = id.split('.', 1)[1]
else:
torrentid = id
table = self.browser.parser.select(self.document.getroot(), 'table.torrent_table')
if len(table) == 0:
table = self.browser.parser.select(self.document.getroot(), 'div.main_column', 1)
is_table = False
else:
table = table[0]
is_table = True
for tr in table.findall('tr' if is_table else 'div'):
if is_table and 'group_torrent' in tr.attrib.get('class', ''):
tds = tr.findall('td')
if not len(tds) == 5:
continue
url = tds[0].find('span').find('a').attrib['href']
m = self.TORRENTID_REGEXP.match(url)
if not m:
warning('ID not found')
continue
if m.group(1) != torrentid:
continue
torrent.url = self.format_url(url)
size, unit = tds[1].text.split()
torrent.size = get_bytes_size(float(size.replace(',', '')), unit)
torrent.seeders = int(tds[3].text)
torrent.leechers = int(tds[4].text)
break
elif not is_table and tr.attrib.get('class', '').startswith('torrent_widget') and \
tr.attrib.get('class', '').endswith('pad'):
url = tr.cssselect('a[title=Download]')[0].attrib['href']
m = self.TORRENTID_REGEXP.match(url)
if not m:
warning('ID not found')
continue
if m.group(1) != torrentid:
continue
torrent.url = self.format_url(url)
size, unit = tr.cssselect('div.details_title strong')[-1].text.strip('()').split()
torrent.size = get_bytes_size(float(size.replace(',', '')), unit)
torrent.seeders = int(tr.cssselect('img[title=Seeders]')[0].tail)
torrent.leechers = int(tr.cssselect('img[title=Leechers]')[0].tail)
break
if not torrent.url:
warning('Torrent %s not found in list' % torrentid)
return None
div = self.parser.select(self.document.getroot(), 'div.main_column', 1)
for box in div.cssselect('div.box'):
title = None
body = None
title_t = box.cssselect('div.head')
if len(title_t) > 0:
title_t = title_t[0]
if title_t.find('strong') is not None:
title_t = title_t.find('strong')
title = title_t.text.strip()
body_t = box.cssselect('div.body,div.desc')
if body_t:
body = html2text(self.parser.tostring(body_t[-1])).strip()
if title and body:
if torrent.description is NotLoaded:
torrent.description = u''
torrent.description += u'%s\n\n%s\n' % (title, body)
divs = self.document.getroot().cssselect('div#files_%s,div#filelist_%s,tr#torrent_%s td' % (torrentid, torrentid, torrentid))
if divs:
torrent.files = []
for div in divs:
table = div.find('table')
if table is None:
continue
for tr in table:
if tr.attrib.get('class', None) != 'colhead_dark':
torrent.files.append(tr.find('td').text)
return torrent
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/gazelle/test.py 0000664 0000000 0000000 00000001735 11666415431 0026661 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class GazelleTest(BackendTest):
BACKEND = 'gazelle'
def test_torrent(self):
l = list(self.backend.iter_torrents('sex'))
if len(l) > 0:
self.backend.get_torrent_file(l[0].id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/geolocip/ 0000775 0000000 0000000 00000000000 11666415431 0025500 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/geolocip/__init__.py 0000664 0000000 0000000 00000000104 11666415431 0027604 0 ustar 00root root 0000000 0000000 from .backend import GeolocIpBackend
__all__ = ['GeolocIpBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/geolocip/backend.py 0000664 0000000 0000000 00000005045 11666415431 0027445 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.geolocip import ICapGeolocIp, IpLocation
from weboob.tools.backend import BaseBackend
from weboob.tools.browser import BaseBrowser, BrowserUnavailable
__all__ = ['GeolocIpBackend']
class GeolocIpBackend(BaseBackend, ICapGeolocIp):
NAME = 'geolocip'
MAINTAINER = 'Julien Veyssier'
EMAIL = 'julien.veyssier@aiur.fr'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = u"IP Addresses geolocalisation with the site www.geolocip.com"
BROWSER = BaseBrowser
def create_default_browser(self):
return self.create_browser()
def get_location(self, ipaddr):
with self.browser:
content = self.browser.readurl('http://www.geolocip.com/?s[ip]=%s&commit=locate+IP!' % str(ipaddr))
if content is None:
raise BrowserUnavailable()
tab = {}
last_line = ''
line = ''
for line in content.split('\n'):
if len(line.split('
')) > 1:
key = last_line.split('
')[1].split('
')[0][0:-2]
value = line.split('
')[1].split('
')[0]
tab[key] = value
last_line = line
iploc = IpLocation(ipaddr)
iploc.city = tab['City']
iploc.region = tab['Region']
iploc.zipcode = tab['Postal code']
iploc.country = tab['Country name']
if tab['Latitude'] != '':
iploc.lt = float(tab['Latitude'])
else:
iploc.lt = 0.0
if tab['Longitude'] != '':
iploc.lg = float(tab['Longitude'])
else:
iploc.lg = 0.0
#iploc.host = 'NA'
#iploc.tld = 'NA'
#iploc.isp = 'NA'
return iploc
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/geolocip/test.py 0000664 0000000 0000000 00000001630 11666415431 0027031 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class GeolocIPTest(BackendTest):
BACKEND = 'geolocip'
def test_geolocip(self):
self.backend.get_location('88.198.11.130')
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/hds/ 0000775 0000000 0000000 00000000000 11666415431 0024455 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/hds/__init__.py 0000664 0000000 0000000 00000000072 11666415431 0026565 0 ustar 00root root 0000000 0000000 from .backend import HDSBackend
__all__ = ['HDSBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/hds/backend.py 0000664 0000000 0000000 00000006422 11666415431 0026422 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.tools.backend import BaseBackend
from weboob.capabilities.messages import ICapMessages, Message, Thread
from .browser import HDSBrowser
__all__ = ['HDSBackend']
class HDSBackend(BaseBackend, ICapMessages):
NAME = 'hds'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = "histoires-de-sexe.net french erotic novels"
STORAGE = {'seen': []}
BROWSER = HDSBrowser
#### ICapMessages ##############################################
def iter_threads(self):
with self.browser:
for story in self.browser.iter_stories():
thread = Thread(story.id)
thread.title = story.title
thread.date = story.date
thread.nb_messages = 1
yield thread
GENDERS = ['', 'boy', 'girl', 'transexual']
def get_thread(self, id):
if isinstance(id, Thread):
thread = id
id = thread.id
else:
thread = None
with self.browser:
story = self.browser.get_story(id)
if not story:
return None
if not thread:
thread = Thread(story.id)
flags = 0
if not thread.id in self.storage.get('seen', default=[]):
flags |= Message.IS_UNREAD
thread.title = story.title
thread.date = story.date
thread.root = Message(thread=thread,
id=0,
title=story.title,
sender=story.author.name,
receivers=None,
date=thread.date,
parent=None,
content=story.body,
children=[],
signature='Written by a %s (%s)' % (self.GENDERS[story.author.sex], story.author.email),
flags=flags)
return thread
def iter_unread_messages(self, thread=None):
for thread in self.iter_threads():
if thread.id in self.storage.get('seen', default=[]):
continue
self.fill_thread(thread, 'root')
yield thread.root
def set_message_read(self, message):
self.storage.set('seen', self.storage.get('seen', default=[]) + [message.thread.id])
self.storage.save()
def fill_thread(self, thread, fields):
return self.get_thread(thread)
OBJECTS = {Thread: fill_thread}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/hds/browser.py 0000664 0000000 0000000 00000004063 11666415431 0026515 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser
from .pages import ValidationPage, HomePage, HistoryPage, StoryPage, AuthorPage
# Browser
class HDSBrowser(BaseBrowser):
ENCODING = 'ISO-8859-1'
DOMAIN = 'histoires-de-sexe.net'
PAGES = {'http://histoires-de-sexe.net/': ValidationPage,
'http://histoires-de-sexe.net/menu.php': HomePage,
'http://histoires-de-sexe.net/sexe/histoires-par-date.php.*': HistoryPage,
'http://histoires-de-sexe.net/sexe.php\?histoire=(?P.+)': StoryPage,
'http://histoires-de-sexe.net/fiche.php\?auteur=(?P.+)': AuthorPage,
}
def iter_stories(self):
self.location('/sexe/histoires-par-date.php')
n = 1
while self.page.get_numerous() == n:
count = 0
for count, story in enumerate(self.page.iter_stories()):
yield story
n += 1
self.location('/sexe/histoires-par-date.php?p=%d' % n)
def get_story(self, id):
id = int(id)
self.location('/sexe.php?histoire=%d' % id)
assert self.is_on_page(StoryPage)
return self.page.get_story()
def get_author(self, name):
self.location(self.buildurl('/fiche.php', auteur=name.encode('iso-8859-15')))
assert self.is_on_page(AuthorPage)
return self.page.get_author()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/hds/pages.py 0000664 0000000 0000000 00000015126 11666415431 0026133 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import datetime
import re
from weboob.tools.browser import BasePage
__all__ = ['ValidationPage', 'HomePage', 'HistoryPage', 'StoryPage']
class ValidationPage(BasePage):
pass
class HomePage(BasePage):
pass
class Author(object):
(UNKNOWN,
MALE,
FEMALE,
TRANSEXUAL) = xrange(4)
def __init__(self, name):
self.name = name
self.sex = self.UNKNOWN
self.email = None
self.description = None
class Story(object):
def __init__(self, id):
self.id = id
self.title = u''
self.date = None
self.category = None
self.author = None
self.body = None
class HistoryPage(BasePage):
def get_numerous(self):
td = self.parser.select(self.document.getroot(), 'td.t0', 1)
n = td.xpath('//u/strong|//u/b')[0].text
return int(n)
def iter_stories(self):
links = self.parser.select(self.document.getroot(), 'a.t11')
story = None
for link in links:
if not story:
m = re.match('.*histoire=(\d+)', link.attrib['href'])
if not m:
self.logger.warning('Unable to parse ID "%s"' % link.attrib['href'])
continue
story = Story(int(m.group(1)))
story.title = link.text.strip()
else:
story.author = Author(link.text.strip())
if not link.tail:
self.logger.warning('There is probably a mistake in the name of %s, skipping...' % story.author.name)
story = None
continue
date_text = link.tail.strip().split('\n')[-1].strip()
m = re.match('(\d+)-(\d+)-(\d+)', date_text)
if not m:
self.logger.warning('Unable to parse datetime "%s"' % date_text)
story = None
continue
story.date = datetime.date(int(m.group(3)),
int(m.group(2)),
int(m.group(1)))
yield story
story = None
class StoryPage(BasePage):
def get_story(self):
p_tags = self.document.getroot().xpath('//body/p')
if len(p_tags) > 0 and p_tags[0].text.strip() == \
u"Le r\xe9cit que vous demandez n'est pas accessible actuellement.":
return None
story = Story((self.group_dict['id']))
story.body = u''
meta = self.parser.select(self.document.getroot(), 'td.t0', 1)
story.author = Author(meta.xpath('./a[@class="t3"]')[0].text.strip())
gender = meta.xpath('./a[@class="t0"]')[0].text
if 'homme' in gender:
story.author.sex = story.author.MALE
elif 'femme' in gender:
story.author.sex = story.author.FEMALE
else:
story.author.sex = story.author.TRANSEXUAL
email_tag = meta.xpath('./span[@class="police1"]')[0]
story.author.email = email_tag.text.strip()
for img in email_tag.findall('img'):
if img.attrib['src'].endswith('meyle1.gif'):
story.author.email += '@'
elif img.attrib['src'].endswith('meyle1pouan.gif'):
story.author.email += '.'
else:
self.logger.warning('Unable to know what image is %s' % img.attrib['src'])
story.author.email += img.tail.strip()
title_tag = self.parser.select(self.document.getroot(), 'h1', 1)
story.title = title_tag.text.strip() if title_tag.text else u''
span = self.parser.select(self.document.getroot(), 'span.t4', 1)
date_text = span.text.strip().split('\n')[-1].strip()
m = re.match('(\d+)-(\d+)-(\d+)', date_text)
if m:
story.date = datetime.date(int(m.group(3)),
int(m.group(2)),
int(m.group(1)))
else:
self.logger.warning('Unable to parse datetime "%s"' % date_text)
story.category = span.find('br').tail.split(':')[1].strip()
div = self.parser.select(self.document.getroot(), 'div[align=justify]', 1)
for para in div.findall('br'):
if para.text is not None:
story.body += para.text.strip()
story.body += '\n'
if para.tail is not None:
story.body += para.tail.strip()
story.body = story.body.replace(u'\x92', "'").strip()
return story
class AuthorPage(BasePage):
def get_author(self):
p_tags = self.document.getroot().xpath('//body/div/font/b')
if len(p_tags) > 0 and p_tags[0].text.strip() == \
u"La fiche de l'auteur n'est plus accessible.":
return None
meta = self.parser.select(self.document.getroot(), 'td.t0', 1)
author_name = meta.xpath('./span[@class="t3"]')[0].text
if author_name is None:
author_name = self.group_dict['name']
author = Author(author_name.strip())
gender = meta.xpath('./a[@class="t0"]')[0].text
if not gender:
author.sex = author.UNKNOWN
elif 'homme' in gender:
author.sex = author.MALE
elif 'femme' in gender:
author.sex = author.FEMALE
else:
author.sex = author.TRANSEXUAL
author.description = u''
for para in meta.getchildren():
if para.tag not in ('b', 'br'):
continue
if para.text is not None:
author.description += '\n\n%s' % para.text.strip()
if para.tail is not None:
author.description += '\n%s' % para.tail.strip()
author.description = author.description.replace(u'\x92', "'").strip()
if author.description.startswith(u'0 récit '):
self.logger.warning('This author does not have published any story.')
return author
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/hds/test.py 0000664 0000000 0000000 00000001763 11666415431 0026015 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
from weboob.tools.misc import limit
__all__ = ['HDSTest']
class HDSTest(BackendTest):
BACKEND = 'hds'
def test_new_messages(self):
for message in limit(self.backend.iter_unread_messages(), 10):
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ina/ 0000775 0000000 0000000 00000000000 11666415431 0024446 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ina/__init__.py 0000664 0000000 0000000 00000000072 11666415431 0026556 0 ustar 00root root 0000000 0000000 from .backend import InaBackend
__all__ = ['InaBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ina/backend.py 0000664 0000000 0000000 00000003665 11666415431 0026421 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.video import ICapVideo
from weboob.tools.backend import BaseBackend
from .browser import InaBrowser
from .video import InaVideo
__all__ = ['InaBackend']
class InaBackend(BaseBackend, ICapVideo):
NAME = 'ina'
MAINTAINER = 'Christophe Benz'
EMAIL = 'christophe.benz@gmail.com'
VERSION = '0.9.1'
DESCRIPTION = 'INA french video archives'
LICENSE = 'AGPLv3+'
BROWSER = InaBrowser
def get_video(self, _id):
return self.browser.get_video(_id)
def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
with self.browser:
return self.browser.iter_search_results(pattern)
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
with self.browser:
video = self.browser.get_video(video.id, video)
if 'thumbnail' in fields and video.thumbnail:
with self.browser:
video.thumbnail.data = self.browser.readurl(video.thumbnail.url)
return video
OBJECTS = {InaVideo: fill_video}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ina/browser.py 0000664 0000000 0000000 00000003136 11666415431 0026506 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser
from weboob.tools.browser.decorators import id2url
from .pages.video import VideoPage, BoutiqueVideoPage
from .pages.search import SearchPage
from .video import InaVideo
__all__ = ['InaBrowser']
class InaBrowser(BaseBrowser):
DOMAIN = 'ina.fr'
PAGES = {'http://boutique\.ina\.fr/video/.+\.html': BoutiqueVideoPage,
'http://www\.ina\.fr/.+\.html': VideoPage,
'http://boutique\.ina\.fr/recherche/.+': SearchPage,
}
@id2url(InaVideo.id2url)
def get_video(self, url, video=None):
self.location(url)
return self.page.get_video(video)
def iter_search_results(self, pattern):
self.location(self.buildurl('http://boutique.ina.fr/recherche/recherche', search=pattern.encode('utf-8')))
assert self.is_on_page(SearchPage)
return self.page.iter_videos()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ina/pages/ 0000775 0000000 0000000 00000000000 11666415431 0025545 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ina/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0027644 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ina/pages/search.py 0000664 0000000 0000000 00000004374 11666415431 0027374 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import datetime
import re
from weboob.tools.browser import BasePage
from weboob.tools.browser import BrokenPageError
from ..video import InaVideo
__all__ = ['SearchPage']
class SearchPage(BasePage):
URL_REGEXP = re.compile(r'/video/(.+)\.html')
def iter_videos(self):
try:
ul = self.parser.select(self.document.getroot(), 'div.container-videos ul', 1)
except BrokenPageError:
# It means there are no results.
return
for li in ul.findall('li'):
id = re.sub(self.URL_REGEXP, r'\1', li.find('a').attrib['href'])
thumbnail = 'http://boutique.ina.fr%s' % li.find('a').find('img').attrib['src']
title = self.parser.select(li, 'p.titre', 1).text
date = self.parser.select(li, 'p.date', 1).text
day, month, year = [int(s) for s in date.split('/')]
date = datetime.datetime(year, month, day)
duration = self.parser.select(li, 'p.duree', 1).text
m = re.match(r'((\d+)h)?((\d+)min)?(\d+)s', duration)
if m:
duration = datetime.timedelta(hours=int(m.group(2) or 0), minutes=int(m.group(4) or 0), seconds=int(m.group(5)))
else:
raise BrokenPageError('Unable to match duration (%r)' % duration)
yield InaVideo('boutique.%s' % id,
title=title,
date=date,
duration=duration,
thumbnail_url=thumbnail,
)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ina/pages/video.py 0000664 0000000 0000000 00000010520 11666415431 0027223 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import datetime
import re
try:
from urlparse import parse_qs
except ImportError:
from cgi import parse_qs
from weboob.tools.browser import BasePage
from weboob.tools.browser import BrokenPageError
from ..video import InaVideo
__all__ = ['VideoPage', 'BoutiqueVideoPage']
class BaseVideoPage(BasePage):
def get_video(self, video):
date, duration = self.get_date_and_duration()
if not video:
video = InaVideo(self.get_id())
video.title = self.get_title()
video.url = self.get_url()
video.date = date
video.duration = duration
video.description = self.get_description()
return video
def get_id(self):
m = self.URL_REGEXP.match(self.url)
if m:
return self.create_id(m.group(1))
self.logger.warning('Unable to parse ID')
return 0
def get_url(self):
qs = parse_qs(self.document.getroot().cssselect('param[name="flashvars"]')[0].attrib['value'])
url = 'http://mp4.ina.fr/lecture/lire/id_notice/%s/token_notice/%s' % (qs['id_notice'][0], qs['token_notice'][0])
return url
def parse_date_and_duration(self, text):
duration_regexp = re.compile('(.* - )?(.+) - ((.+)h)?((.+)min)?(.+)s')
m = duration_regexp.match(text)
if m:
day, month, year = [int(s) for s in m.group(2).split('/')]
date = datetime.datetime(year, month, day)
duration = datetime.timedelta(hours=int(m.group(4) if m.group(4) is not None else 0),
minutes=int(m.group(6) if m.group(6) is not None else 0),
seconds=int(m.group(7)))
return date, duration
else:
raise BrokenPageError('Unable to parse date and duration')
def create_id(self, id):
raise NotImplementedError()
def get_date_and_duration(self):
raise NotImplementedError()
def get_title(self):
raise NotImplementedError()
def get_description(self):
raise NotImplementedError()
class VideoPage(BaseVideoPage):
URL_REGEXP = re.compile('http://www.ina.fr/(.+)\.html')
def create_id(self, id):
return u'www.%s' % id
def get_date_and_duration(self):
qr = self.parser.select(self.document.getroot(), 'div.container-global-qr')[0].find('div').findall('div')[1]
return self.parse_date_and_duration(qr.find('h2').tail.strip())
def get_title(self):
qr = self.parser.select(self.document.getroot(), 'div.container-global-qr')[0].find('div').findall('div')[1]
return qr.find('h2').text.strip()
def get_description(self):
return self.parser.select(self.document.getroot(), 'div.container-global-qr')[1].find('div').find('p').text.strip()
class BoutiqueVideoPage(BaseVideoPage):
URL_REGEXP = re.compile('http://boutique.ina.fr/video/(.+).html')
def create_id(self, id):
return u'boutique.%s' % id
def get_description(self):
el = self.document.getroot().cssselect('div.bloc-produit-haut div.contenu p')[0]
if el is not None:
return el.text.strip()
def get_date_and_duration(self):
el = self.document.getroot().cssselect('div.bloc-produit-haut p.date')[0]
if el is not None:
return self.parse_date_and_duration(el.text.strip())
else:
raise BrokenPageError('Unable to find date and duration element')
def get_title(self):
el = self.document.getroot().cssselect('div.bloc-produit-haut h1')[0]
if el is not None:
return unicode(el.text.strip())
else:
return None
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ina/test.py 0000664 0000000 0000000 00000002217 11666415431 0026001 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class INATest(BackendTest):
BACKEND = 'ina'
def test_ina(self):
l = list(self.backend.iter_search_results('chirac'))
self.assertTrue(len(l) > 0)
v = l[0]
self.backend.fillobj(v, ('url',))
self.assertTrue(v.url and v.url.startswith('http://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
self.backend.browser.openurl(v.url)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ina/video.py 0000664 0000000 0000000 00000002172 11666415431 0026130 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.video import BaseVideo
__all__ = ['InaVideo']
class InaVideo(BaseVideo):
@classmethod
def id2url(cls, _id):
if not '.' in _id:
return None
site, _id = _id.split('.', 1)
if site == 'boutique':
return 'http://boutique.ina.fr/video/%s.html' % _id
if site == 'www':
return 'http://www.ina.fr/%s.html' % _id
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/inrocks/ 0000775 0000000 0000000 00000000000 11666415431 0025347 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/inrocks/__init__.py 0000664 0000000 0000000 00000001521 11666415431 0027457 0 ustar 00root root 0000000 0000000 "init of NewspaperInrocksBackend"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import NewspaperInrocksBackend
__all__ = ['NewspaperInrocksBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/inrocks/backend.py 0000664 0000000 0000000 00000002605 11666415431 0027313 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
"backend for http://www.lesinrocks.com"
from weboob.capabilities.messages import ICapMessages
from weboob.tools.capabilities.messages.GenericBackend import GenericNewspaperBackend
from .browser import NewspaperInrocksBrowser
from .tools import rssid
class NewspaperInrocksBackend(GenericNewspaperBackend, ICapMessages):
"NewspaperInrocksBackend class"
MAINTAINER = 'Julien Hebert'
EMAIL = 'juke@free.fr'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
STORAGE = {'seen': {}}
NAME = 'inrocks'
DESCRIPTION = u'Inrock French news website'
BROWSER = NewspaperInrocksBrowser
RSS_FEED = 'http://www.lesinrocks.com/fileadmin/rss/actus.xml'
RSSID = rssid
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/inrocks/browser.py 0000664 0000000 0000000 00000002706 11666415431 0027411 0 ustar 00root root 0000000 0000000 "browser for inrocks.fr website"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .pages.article import ArticlePage
from .pages.inrockstv import InrocksTvPage
from weboob.tools.browser import BaseBrowser
class NewspaperInrocksBrowser(BaseBrowser):
"NewspaperInrocksBrowser class"
PAGES = {
'http://www.lesinrocks.com/(?!inrockstv).+/.*': ArticlePage,
'http://www.lesinrocks.com/inrockstv/.*': InrocksTvPage,
'http://blogs.lesinrocks.com/.*': ArticlePage,
}
def is_logged(self):
return False
def login(self):
pass
def fillobj(self, obj, fields):
pass
def get_content(self, _id):
"return page article content"
self.location(_id)
return self.page.get_article(_id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/inrocks/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026446 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/inrocks/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0030545 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/inrocks/pages/article.py 0000664 0000000 0000000 00000006465 11666415431 0030456 0 ustar 00root root 0000000 0000000 "ArticlePage object for inrocks"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BrokenPageError
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage, try_remove, \
try_remove_from_selector_list, \
drop_comments, NoneMainDiv
class ArticlePage(GenericNewsPage):
"ArticlePage object for inrocks"
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_title_selector = "h1"
self.element_author_selector = "div.name>span"
self.element_body_selector = "div.maincol"
def get_body(self):
try :
element_body = self.get_element_body()
except NoneMainDiv:
return None
else:
div_header_element = self.parser.select(element_body, "div.header", 1)
element_detail = self.parser.select(element_body, "div.details", 1)
div_content_element = self.parser.select(element_body, "div.content", 1)
drop_comments(element_body)
try_remove(self.parser, element_body, "div.sidebar")
try_remove(self.parser, element_detail, "div.footer")
try_remove_from_selector_list(self.parser,
div_header_element,
["h1", "div.picture", "div.date",
"div.news-single-img",
"div.metas_img", "strong"])
try_remove_from_selector_list(self.parser,
div_content_element,
["div.tw_button", "div.wpfblike"])
try :
description_element = self.parser.select(div_header_element,
"div.description", 1)
except BrokenPageError:
pass
else:
text_content = description_element.text_content()
if len(text_content.strip()) == 0 :
description_element.drop_tree()
else:
if len(description_element) == 1:
description_element.drop_tag()
if len(div_header_element.text_content().strip()) == 0:
div_header_element.drop_tree()
if len(div_header_element) == 1:
div_header_element.drop_tag()
if len(element_detail) == 1:
element_detail.drop_tag()
div_content_element.drop_tag()
return self.parser.tostring(element_body)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/inrocks/pages/inrockstv.py 0000664 0000000 0000000 00000002365 11666415431 0031050 0 ustar 00root root 0000000 0000000 "ArticlePage object for inrocks"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage
class InrocksTvPage(GenericNewsPage):
"ArticlePage object for inrocks"
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_title_selector = "h2"
self.element_author_selector = "div.name>span"
self.element_body_selector = "span.infos"
def get_body(self):
element_body = self.get_element_body()
return self.parser.tostring(element_body)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/inrocks/test.py 0000664 0000000 0000000 00000001720 11666415431 0026700 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
__all__ = ['InrocksTest']
class InrocksTest(BackendTest):
BACKEND = 'inrocks'
def test_new_messages(self):
for message in self.backend.iter_unread_messages():
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/inrocks/tools.py 0000664 0000000 0000000 00000002425 11666415431 0027064 0 ustar 00root root 0000000 0000000 "common tools for inrocks backend"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
def id2url(_id):
"return an url from an id"
regexp2 = re.compile("(\w+).([0-9]+).(.*$)")
match = regexp2.match(_id)
if match:
return 'http://www.20minutes.fr/%s/%s/%s' % ( match.group(1),
match.group(2),
match.group(3))
else:
raise ValueError("id doesn't match")
def url2id(url):
"return an id from an url"
return url
def rssid(entry):
return url2id(entry.id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ipinfodb/ 0000775 0000000 0000000 00000000000 11666415431 0025471 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ipinfodb/__init__.py 0000664 0000000 0000000 00000000104 11666415431 0027575 0 ustar 00root root 0000000 0000000 from .backend import IpinfodbBackend
__all__ = ['IpinfodbBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ipinfodb/backend.py 0000664 0000000 0000000 00000007005 11666415431 0027434 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.geolocip import ICapGeolocIp, IpLocation
from weboob.tools.backend import BaseBackend
from weboob.tools.browser import BaseBrowser, BrowserUnavailable
__all__ = ['IpinfodbBackend']
class IpinfodbBackend(BaseBackend, ICapGeolocIp):
NAME = 'ipinfodb'
MAINTAINER = 'Julien Veyssier'
EMAIL = 'julien.veyssier@aiur.fr'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = u"IP Addresses geolocalisation with the site ipinfodb.com"
BROWSER = BaseBrowser
def create_default_browser(self):
return self.create_browser()
def get_location(self, ipaddr):
with self.browser:
content = self.browser.readurl('http://ipinfodb.com/ip_locator.php?ip=%s' % str(ipaddr))
if content is None:
raise BrowserUnavailable()
if 'Invalid IP or domain name' in content:
raise Exception('Bad parameter')
else:
tab = {'City' : 'NA' ,\
'Country name' : 'NA' ,\
'Region' : 'NA' ,\
'Latitude' : 'NA' ,\
'Longitude' : 'NA' ,\
'hostname' : 'NA' ,\
'zipcode' : 'NA'}
line = ''
for line in content.split('\n'):
if '
' in line:
if 'Country :' in line:
tab['Country name'] = line.split('Country : ')[1].split('<')[0]
elif "State/Province :" in line:
tab['Region'] = line.split('State/Province : ')[1].split('<')[0]
elif "City :" in line:
tab['City'] = line.split('City : ')[1].split('<')[0]
elif "Latitude :" in line:
tab['Latitude'] = line.split('Latitude : ')[1].split('<')[0]
elif "Longitude :" in line:
tab['Longitude'] = line.split('Longitude : ')[1].split('<')[0]
elif "Hostname :" in line:
tab['hostname'] = line.split('Hostname : ')[1].split('<')[0]
iploc = IpLocation(ipaddr)
iploc.city = tab['City'].decode('utf-8')
iploc.region = tab['Region']
iploc.zipcode = tab['zipcode']
iploc.country = tab['Country name']
try :
iploc.lt = float(tab['Latitude'])
iploc.lg = float(tab['Longitude'])
except ValueError:
pass
iploc.host = tab['hostname']
iploc.tld = tab['hostname'].split('.')[-1]
#iploc.isp = 'NA'
return iploc
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ipinfodb/test.py 0000664 0000000 0000000 00000001722 11666415431 0027024 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class IpinfodbTest(BackendTest):
BACKEND = 'ipinfodb'
def test_ipinfobd(self):
self.backend.get_location('88.198.11.130')
self.backend.get_location('2a01:4f8:130:3062::1')
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/isohunt/ 0000775 0000000 0000000 00000000000 11666415431 0025370 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/isohunt/__init__.py 0000664 0000000 0000000 00000000102 11666415431 0027472 0 ustar 00root root 0000000 0000000 from .backend import IsohuntBackend
__all__ = ['IsohuntBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/isohunt/backend.py 0000664 0000000 0000000 00000003144 11666415431 0027333 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.torrent import ICapTorrent
from weboob.tools.backend import BaseBackend
from .browser import IsohuntBrowser
__all__ = ['IsohuntBackend']
class IsohuntBackend(BaseBackend, ICapTorrent):
NAME = 'isohunt'
MAINTAINER = 'Julien Veyssier'
EMAIL = 'julien.veyssier@aiur.fr'
VERSION = '0.9.1'
DESCRIPTION = 'isohunt.com bittorrent tracker'
LICENSE = 'AGPLv3+'
BROWSER = IsohuntBrowser
def create_default_browser(self):
return self.create_browser()
def get_torrent(self, id):
return self.browser.get_torrent(id)
def get_torrent_file(self, id):
torrent = self.browser.get_torrent(id)
if not torrent:
return None
return self.browser.openurl(torrent.url.encode('utf-8')).read()
def iter_torrents(self, pattern):
return self.browser.iter_torrents(pattern.replace(' ','+'))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/isohunt/browser.py 0000664 0000000 0000000 00000003265 11666415431 0027433 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser
from .pages.torrents import TorrentsPage, TorrentPage
__all__ = ['IsohuntBrowser']
class IsohuntBrowser(BaseBrowser):
DOMAIN = 'isohunt.com'
PROTOCOL = 'https'
ENCODING = 'utf-8'
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
PAGES = {
'https://isohunt.com/torrents/.*iht=-1&ihp=1&ihs1=1&iho1=d' : TorrentsPage,
'https://isohunt.com/torrent_details.*tab=summary' : TorrentPage,
}
def home(self):
return self.location('https://isohunt.com')
def iter_torrents(self, pattern):
self.location('https://isohunt.com/torrents/%s?iht=-1&ihp=1&ihs1=1&iho1=d' % pattern.encode('utf-8'))
assert self.is_on_page(TorrentsPage)
return self.page.iter_torrents()
def get_torrent(self, id):
self.location('https://isohunt.com/torrent_details/%s/?tab=summary' % id)
assert self.is_on_page(TorrentPage)
return self.page.get_torrent(id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/isohunt/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026467 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/isohunt/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0030566 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/isohunt/pages/torrents.py 0000664 0000000 0000000 00000011702 11666415431 0030722 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.torrent import Torrent
from weboob.tools.browser import BasePage
from weboob.tools.misc import get_bytes_size
__all__ = ['TorrentsPage']
class TorrentsPage(BasePage):
def iter_torrents(self):
for tr in self.document.getiterator('tr'):
if tr.attrib.get('class', '') == 'hlRow':
# sometimes the first tr also has the attribute hlRow
# i use that to ditinct it from the others
if tr.attrib.has_key('onmouseout'):
atitle = tr.getchildren()[2].getchildren()[1]
title = atitle.text
if not title:
title = ''
for bold in atitle.getchildren():
if bold.text:
title += bold.text
if bold.tail:
title += bold.tail
idt = tr.getchildren()[2].getchildren()[0].attrib.get('href','')
idt = idt.split('/')[2]
size = tr.getchildren()[3].text
u = size[-2:]
size = float(size[:-3])
seed = tr.getchildren()[4].text
leech = tr.getchildren()[5].text
url = 'https://isohunt.com/download/%s/mon_joli_torrent.torrent' % idt
yield Torrent(idt,
title,
url=url,
size=get_bytes_size(size, u),
seeders=int(seed),
leechers=int(leech))
class TorrentPage(BasePage):
def get_torrent(self, id):
title = ''
url = 'https://isohunt.com/download/%s/%s.torrent' % (id, id)
for a in self.document.getiterator('a'):
if 'Search more torrents of' in a.attrib.get('title', ''):
title = a.tail
seed = -1
leech = -1
tip_id = "none"
for span in self.document.getiterator('span'):
if span.attrib.get('style', '') == 'color:green;' and ('ShowTip' in span.attrib.get('onmouseover', '')):
seed = span.tail.split(' ')[1]
tip_id = span.attrib.get('onmouseover', '').split("'")[1]
for div in self.document.getiterator('div'):
# find the corresponding super tip which appears on super mouse hover!
if div.attrib.get('class', '') == 'dirs ydsf' and tip_id in div.attrib.get('id', ''):
leech = div.getchildren()[0].getchildren()[1].tail.split(' ')[2]
# the with the size in it doesn't have a distinction
# have to get it by higher
elif div.attrib.get('id', '') == 'torrent_details':
size = div.getchildren()[6].getchildren()[0].getchildren()[0].text
u = size[-2:]
size = float(size[:-3])
# files and description (uploader's comment)
description = 'No description'
files = []
count_p_found = 0
for p in self.document.getiterator('p'):
if p.attrib.get('style', '') == 'line-height:1.2em;margin-top:1.8em':
count_p_found += 1
if count_p_found == 1:
if p.getchildren()[1].tail != None:
description = p.getchildren()[1].tail
if count_p_found == 2:
if p.getchildren()[0].text == 'Directory:':
files.append(p.getchildren()[0].tail.strip()+'/')
else:
files.append(p.getchildren()[0].tail.strip())
for td in self.document.getiterator('td'):
if td.attrib.get('class', '') == 'fileRows':
filename = td.text
for slash in td.getchildren():
filename += '/'
filename += slash.tail
files.append(filename)
#--------------------------TODO
torrent = Torrent(id, title)
torrent.url = url
torrent.size = get_bytes_size(size, u)
torrent.seeders = int(seed)
torrent.leechers = int(leech)
torrent.description = description
torrent.files = files
return torrent
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/isohunt/test.py 0000664 0000000 0000000 00000001742 11666415431 0026725 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class IsohuntTest(BackendTest):
BACKEND = 'isohunt'
def test_torrent(self):
l = list(self.backend.iter_torrents('debian'))
if len(l) > 0:
self.backend.get_torrent_file(l[0].id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/izneo/ 0000775 0000000 0000000 00000000000 11666415431 0025023 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/izneo/__init__.py 0000664 0000000 0000000 00000001437 11666415431 0027141 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import IzneoBackend
__all__ = ['IzneoBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/izneo/backend.py 0000664 0000000 0000000 00000006157 11666415431 0026775 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
try:
import simplejson as json
except ImportError:
import json
import re
from weboob.capabilities.gallery import ICapGallery, BaseGallery, BaseImage
from weboob.tools.backend import BaseBackend
from weboob.tools.browser import BaseBrowser, BasePage
__all__ = ['IzneoBackend']
class ReaderV2(BasePage):
def get_ean(self):
return self.document.xpath("//div[@id='viewer']/attribute::rel")[0]
def iter_gallery_images(self, gallery):
ean = self.get_ean()
pages = json.load(self.browser.openurl(
'http://www.izneo.com/playerv2/ajax.php?ean=%s&action=get_list_jpg'
% ean))
for page in pages:
width = 1200 # maximum width
yield BaseImage(page['page'],
gallery=gallery,
url=("http://www.izneo.com/playerv2/%s/%s/%s/%d/%s" %
(page['expires'], page['token'], ean, width, page['page'])))
class IzneoBrowser(BaseBrowser):
PAGES = { r'http://.+\.izneo.\w+/readv2-.+': ReaderV2 }
def iter_gallery_images(self, gallery):
self.location(gallery.url)
assert self.is_on_page(ReaderV2)
return self.page.iter_gallery_images(gallery)
def fill_image(self, image, fields):
if 'data' in fields:
image.data = self.readurl(self.request_class(
image.url, None, {'Referer': image.gallery.url}))
class IzneoBackend(BaseBackend, ICapGallery):
NAME = 'izneo'
MAINTAINER = 'Roger Philibert'
EMAIL = 'roger.philibert@gmail.com'
VERSION = '0.9.1'
DESCRIPTION = 'Izneo'
LICENSE = 'AGPLv3+'
BROWSER = IzneoBrowser
def iter_gallery_images(self, gallery):
with self.browser:
return self.browser.iter_gallery_images(gallery)
def get_gallery(self, _id):
match = re.match(r'(?:(?:.+izneo.com/)?readv2-)?(\d+-\d+)/?$', _id)
if match is None:
return None
_id = match.group(1)
gallery = BaseGallery(_id, url=('http://www.izneo.com/readv2-%s' % _id))
with self.browser:
return gallery
def fill_gallery(self, gallery, fields):
gallery.title = gallery.id
def fill_image(self, image, fields):
with self.browser:
self.browser.fill_image(image, fields)
OBJECTS = {
BaseGallery: fill_gallery,
BaseImage: fill_image }
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/kickass/ 0000775 0000000 0000000 00000000000 11666415431 0025327 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/kickass/__init__.py 0000664 0000000 0000000 00000000102 11666415431 0027431 0 ustar 00root root 0000000 0000000 from .backend import KickassBackend
__all__ = ['KickassBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/kickass/backend.py 0000664 0000000 0000000 00000003153 11666415431 0027272 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.torrent import ICapTorrent
from weboob.tools.backend import BaseBackend
from .browser import KickassBrowser
__all__ = ['KickassBackend']
class KickassBackend(BaseBackend, ICapTorrent):
NAME = 'kickass'
MAINTAINER = 'Julien Veyssier'
EMAIL = 'julien.veyssier@aiur.fr'
VERSION = '0.9.1'
DESCRIPTION = 'kickasstorrent.com bittorrent tracker'
LICENSE = 'AGPLv3+'
BROWSER = KickassBrowser
def create_default_browser(self):
return self.create_browser()
def get_torrent(self, id):
return self.browser.get_torrent(id)
def get_torrent_file(self, id):
torrent = self.browser.get_torrent(id)
if not torrent:
return None
return self.browser.openurl(torrent.url.encode('utf-8')).read()
def iter_torrents(self, pattern):
return self.browser.iter_torrents(pattern.replace(' ','+'))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/kickass/browser.py 0000664 0000000 0000000 00000003305 11666415431 0027365 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser
from .pages.torrents import TorrentsPage, TorrentPage
__all__ = ['KickassBrowser']
class KickassBrowser(BaseBrowser):
DOMAIN = 'kickasstorrents.com'
PROTOCOL = 'http'
ENCODING = 'utf-8'
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
PAGES = {
'http://fr.(kickasstorrents.com|kat.ph)/new/.*field=seeders&sorder=desc': TorrentsPage,
'http://fr.(kickasstorrents.com|kat.ph)/.*.html': TorrentPage,
}
def home(self):
return self.location('http://kickasstorrents.com')
def iter_torrents(self, pattern):
self.location('http://fr.kickasstorrents.com/new/?q=%s&field=seeders&sorder=desc' % pattern.encode('utf-8'))
assert self.is_on_page(TorrentsPage)
return self.page.iter_torrents()
def get_torrent(self, id):
self.location('http://fr.kickasstorrents.com/%s.html' % id)
assert self.is_on_page(TorrentPage)
return self.page.get_torrent(id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/kickass/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026426 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/kickass/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0030525 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/kickass/pages/torrents.py 0000664 0000000 0000000 00000011574 11666415431 0030670 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier, Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
try:
from urlparse import parse_qs
except ImportError:
from cgi import parse_qs
from urlparse import urlsplit
from weboob.capabilities.torrent import Torrent
from weboob.tools.browser import BasePage
from weboob.tools.misc import get_bytes_size
__all__ = ['TorrentsPage']
class TorrentsPage(BasePage):
def iter_torrents(self):
for tr in self.document.getiterator('tr'):
if tr.attrib.get('class', '') == 'odd' or tr.attrib.get('class', '') == ' even':
if not 'id' in tr.attrib:
continue
title = tr.getchildren()[0].getchildren()[1].getchildren()[1].text
if not title:
title = ''
for red in tr.getchildren()[0].getchildren()[1].getchildren()[1].getchildren():
title += red.text_content()
idt = tr.getchildren()[0].getchildren()[1].getchildren()[1].attrib.get('href', '').replace('/', '') \
.replace('.html', '')
# look for url
for a in tr.getchildren()[0].getiterator('a'):
if '.torrent' in a.attrib.get('href', ''):
url = a.attrib['href']
size = tr.getchildren()[1].text
u = tr.getchildren()[1].getchildren()[0].text
size = size = size.replace(',', '.')
size = float(size)
seed = tr.getchildren()[4].text
leech = tr.getchildren()[5].text
yield Torrent(idt,
title,
url=url,
filename=parse_qs(urlsplit(url).query).get('title', [None])[0],
size=get_bytes_size(size, u),
seeders=int(seed),
leechers=int(leech))
class TorrentPage(BasePage):
def get_torrent(self, id):
seed = 0
leech = 0
description = 'No description'
url = 'No Url found'
for div in self.document.getiterator('div'):
if div.attrib.get('id', '') == 'desc':
try:
description = div.text_content()
except UnicodeDecodeError:
description = 'Description with invalid UTF-8.'
elif div.attrib.get('class', '') == 'seedBlock':
if div.getchildren()[1].text is not None:
seed = int(div.getchildren()[1].text)
else:
seed = 0
elif div.attrib.get('class', '') == 'leechBlock':
if div.getchildren()[1].text is not None:
leech = int(div.getchildren()[1].text)
else:
leech = 0
for h in self.document.getiterator('h1'):
if h.attrib.get('class', '') == 'torrentName':
title = h.getchildren()[0].getchildren()[0].text
for a in self.document.getiterator('a'):
if ('Download' in a.attrib.get('title', '')) and ('torrent file' in a.attrib.get('title', '')):
url = a.attrib.get('href', '')
size = 0
for span in self.document.getiterator('span'):
# sometimes there are others span, this is not so sure but the size of the children list
# is enough to know if this is the right span
if (span.attrib.get('class', '') == 'folder' or span.attrib.get('class', '') == 'folderopen') and len(span.getchildren())>2:
size = span.getchildren()[1].tail
u = span.getchildren()[2].text
size = float(size.split(': ')[1].replace(',', '.'))
files = []
for td in self.document.getiterator('td'):
if td.attrib.get('class', '') == 'torFileName':
files.append(td.text)
torrent = Torrent(id, title)
torrent.url = url
torrent.filename = parse_qs(urlsplit(url).query).get('title', [None])[0]
torrent.size = get_bytes_size(size, u)
torrent.seeders = int(seed)
torrent.leechers = int(leech)
torrent.description = description
torrent.files = files
return torrent
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/kickass/test.py 0000664 0000000 0000000 00000003550 11666415431 0026663 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier, Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
from weboob.capabilities.base import NotLoaded
import urllib
from random import choice
class KickassTest(BackendTest):
BACKEND = 'kickass'
def test_torrent(self):
torrents = list(self.backend.iter_torrents('debian'))
for torrent in torrents:
path, qs = urllib.splitquery(torrent.url)
assert path.endswith('.torrent')
if qs:
assert torrent.filename
assert torrent.id
assert torrent.name
assert torrent.description is NotLoaded
full_torrent = self.backend.get_torrent(torrent.id)
# do not assert torrent.name is full_torrent.name
# (or even that one contains another), it isn't always true!
assert full_torrent.name
assert full_torrent.url
assert full_torrent.description is not NotLoaded
# get the file of a random torrent
# from the list (getting them all would be too long)
if len(torrents):
torrent = choice(torrents)
self.backend.get_torrent_file(torrent.id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lcl/ 0000775 0000000 0000000 00000000000 11666415431 0024451 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lcl/__init__.py 0000664 0000000 0000000 00000001432 11666415431 0026562 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import LCLBackend
__all__ = ['LCLBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lcl/backend.py 0000664 0000000 0000000 00000004551 11666415431 0026417 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Pierre Mazière
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.bank import ICapBank, AccountNotFound
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import ValueBackendPassword, Value
from .browser import LCLBrowser
__all__ = ['LCLBackend']
class LCLBackend(BaseBackend, ICapBank):
NAME = 'lcl'
MAINTAINER = u'Pierre Mazière'
EMAIL = 'pierre.maziere@gmail.com'
VERSION = '0.9.1'
DESCRIPTION = 'Le Credit Lyonnais crappy french bank'
LICENSE = 'AGPLv3+'
CONFIG = BackendConfig(ValueBackendPassword('login', label='Account ID', regexp='^\d{1,6}\w$', masked=False),
ValueBackendPassword('password', label='Password of account'),
Value('agency', label='Agency code', regexp='^\d{3,4}$'))
BROWSER = LCLBrowser
def create_default_browser(self):
return self.create_browser(self.config['agency'].get(),
self.config['login'].get(),
self.config['password'].get())
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
with self.browser:
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
def iter_operations(self, account):
""" TODO Not supported yet """
return iter([])
def iter_history(self, account):
with self.browser:
for history in self.browser.get_history(account):
yield history
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lcl/browser.py 0000664 0000000 0000000 00000006547 11666415431 0026522 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Pierre Mazière
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from .pages import LoginPage, LoginResultPage, FramePage, AccountsPage, AccountHistoryPage
__all__ = ['LCLBrowser']
# Browser
class LCLBrowser(BaseBrowser):
PROTOCOL = 'https'
DOMAIN = 'particuliers.secure.lcl.fr'
ENCODING = 'utf-8'
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
PAGES = {
'https://particuliers.secure.lcl.fr/index.html': LoginPage,
'https://particuliers.secure.lcl.fr/everest/UWBI/UWBIAccueil\?DEST=IDENTIFICATION': LoginResultPage,
'https://particuliers.secure.lcl.fr/outil/UWSP/Synthese/accesSynthese': AccountsPage,
'https://particuliers.secure.lcl.fr/outil/UWB2/Accueil\?DEST=INIT': FramePage,
'https://particuliers.secure.lcl.fr/outil/UWLM/ListeMouvementsPar/accesListeMouvementsPar.*': AccountHistoryPage,
}
def __init__(self, agency, *args, **kwargs):
self.agency = agency
BaseBrowser.__init__(self, *args, **kwargs)
def is_logged(self):
return self.page and not self.is_on_page(LoginPage)
def login(self):
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
assert self.password.isdigit()
assert isinstance(self.agency, basestring)
assert self.agency.isdigit()
if not self.is_on_page(LoginPage):
self.location('%s://%s/index.html' % (self.PROTOCOL, self.DOMAIN),\
no_login=True)
if not self.page.login(self.agency, self.username, self.password) or \
not self.is_logged() or \
(self.is_on_page(LoginResultPage) and self.page.is_error()) :
raise BrowserIncorrectPassword()
self.location('%s://%s/outil/UWSP/Synthese/accesSynthese' \
% (self.PROTOCOL, self.DOMAIN))
def get_accounts_list(self):
if not self.is_on_page(AccountsPage):
self.login()
return self.page.get_list()
def get_account(self, id):
assert isinstance(id, basestring)
l = self.get_accounts_list()
for a in l:
if a.id == id:
return a
return None
def get_history(self,account):
if not self.is_on_page(AccountHistoryPage) :
self.location('%s://%s%s' % (self.PROTOCOL, self.DOMAIN, account.link_id))
return self.page.get_operations()
#def get_coming_operations(self, account):
# if not self.is_on_page(AccountComing) or self.page.account.id != account.id:
# self.location('/NS_AVEEC?ch4=%s' % account.link_id)
# return self.page.get_operations()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lcl/pages.py 0000664 0000000 0000000 00000007737 11666415431 0026140 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Pierre Mazière
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import date
from weboob.capabilities.bank import Operation
from weboob.capabilities.bank import Account
from weboob.tools.browser import BasePage, BrowserUnavailable
class LoginPage(BasePage):
def login(self, agency, login, passwd):
self.browser.select_form(nr=0)
self.browser['agenceId'] = agency
self.browser['compteId'] = login
self.browser['CodeId'] = passwd
try:
self.browser.submit()
except BrowserUnavailable:
# Login is not valid
return False
return True
class LoginResultPage(BasePage):
def is_error(self):
for text in self.document.find('body').itertext():
text=text.strip()
# Login seems valid, but password does not
needle='Les données saisies sont incorrectes'
if text.startswith(needle.decode('utf-8')):
return True
return False
class FramePage(BasePage):
pass
class AccountsPage(BasePage):
def get_list(self):
l = []
for div in self.document.getiterator('div'):
if div.attrib.get('class')=="unCompte-CD" or div.attrib.get('class')=="unCompte-CE":
account = Account()
account.id = div.attrib.get('id').replace('-','')
for td in div.getiterator('td'):
if td.find("div") is not None and td.find("div").attrib.get('class') == 'libelleCompte':
account.label = td.find("div").text
elif td.find('a') is not None and td.find('a').attrib.get('class') is None:
balance = td.find('a').text.replace(u"\u00A0",'').replace('.','').replace('+','').replace(',','.')
account.balance = float(balance)
account.link_id = td.find('a').attrib.get('href')
l.append(account)
return l
class AccountHistoryPage(BasePage):
def on_loaded(self):
self.operations = []
for td in self.document.iter('td'):
text=td.findtext("b")
if text is None:
continue
prefix='Solde au'
if text.startswith(prefix.decode('utf-8')):
table=td.getparent().getparent()
for tr in table.iter('tr'):
tr_class=tr.attrib.get('class')
if tr_class == 'tbl1' or tr_class=='tbl2':
tds=tr.findall('td')
d=date(*reversed([int(x) for x in tds[0].text.split('/')]))
label=u''+tds[1].find('a').text.strip()
if tds[2].text.strip() != u"":
amount = - float(tds[2].text.strip().replace('.','').replace(',','.').replace(u"\u00A0",'').replace(' ',''))
else:
amount= float(tds[3].text.strip().replace('.','').replace(',','.').replace(u"\u00A0",'').replace(' ',''))
operation=Operation(len(self.operations))
operation.date=d
operation.label=label
operation.amount=amount
self.operations.append(operation)
def get_operations(self):
return self.operations
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lcl/test.py 0000664 0000000 0000000 00000001600 11666415431 0025777 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class LCLtTest(BackendTest):
BACKEND = 'lcl'
def test_lcl(self):
list(self.backend.iter_accounts())
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lefigaro/ 0000775 0000000 0000000 00000000000 11666415431 0025467 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lefigaro/__init__.py 0000664 0000000 0000000 00000001514 11666415431 0027601 0 ustar 00root root 0000000 0000000 "NewspaperFigaroBackend init"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import NewspaperFigaroBackend
__all__ = ['NewspaperFigaroBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lefigaro/backend.py 0000664 0000000 0000000 00000002602 11666415431 0027430 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
"backend for http://www.lefigaro.fr"
from weboob.capabilities.messages import ICapMessages
from weboob.tools.capabilities.messages.GenericBackend import GenericNewspaperBackend
from .browser import NewspaperFigaroBrowser
from .tools import rssid
class NewspaperFigaroBackend(GenericNewspaperBackend, ICapMessages):
"NewspaperFigaroBackend class"
MAINTAINER = 'Julien Hebert'
EMAIL = 'juke@free.fr'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
STORAGE = {'seen': {}}
NAME = 'lefigaro'
DESCRIPTION = u'Lefigaro French news website'
BROWSER = NewspaperFigaroBrowser
RSS_FEED = 'http://rss.lefigaro.fr/lefigaro/laune?format=xml'
RSSID = rssid
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lefigaro/browser.py 0000664 0000000 0000000 00000003630 11666415431 0027526 0 ustar 00root root 0000000 0000000 "browser for lefigaro website"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .pages.article import ArticlePage
from .pages.flashactu import FlashActuPage
from .pages.special import SpecialPage
from weboob.tools.browser import BaseBrowser, BasePage
class IndexPage(BasePage):
pass
class NewspaperFigaroBrowser(BaseBrowser):
"NewspaperFigaroBrowser class"
PAGES = {
"http://\w+.lefigaro.fr/flash-.*/(\d{4})/(\d{2})/(\d{2})/(.*$)": FlashActuPage,
"http://\w+.lefigaro.fr/bd/(\d{4})/(\d{2})/(\d{2})/(.*$)": FlashActuPage,
"http://\w+.lefigaro.fr/actualite/(\d{4})/(\d{2})/(\d{2})/(.*$)": SpecialPage,
"http://\w+.lefigaro.fr/(?!flash-|bd|actualite).+/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
"http://\w+.lefigaro.fr/actualite-.*/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
"http://\w+.lefigaro.fr/": IndexPage,
}
def is_logged(self):
return False
def login(self):
pass
def fillobj(self, obj, fields):
pass
def get_content(self, _id):
"return page article content"
self.location(_id)
if self.is_on_page(IndexPage):
return None
return self.page.get_article(_id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lefigaro/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026566 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lefigaro/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0030665 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lefigaro/pages/article.py 0000664 0000000 0000000 00000003413 11666415431 0030564 0 ustar 00root root 0000000 0000000 "ArticlePage object for inrocks"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage, remove_from_selector_list, drop_comments, try_drop_tree, try_remove_from_selector_list
class ArticlePage(GenericNewsPage):
"ArticlePage object for inrocks"
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_title_selector = "h1"
self.element_author_selector = "div.name>span"
self.element_body_selector = "#article"
def get_body(self):
element_body = self.get_element_body()
remove_from_selector_list(self.parser, element_body, [self.element_title_selector])
drop_comments(element_body)
try_drop_tree(self.parser, element_body, "script")
try_remove_from_selector_list(self.parser, element_body, ["div.infos", "div.photo", "div.art_bandeau_bottom", "div.view", "span.auteur_long", "#toolsbar", 'link'])
element_body.find_class("texte")[0].drop_tag()
element_body.tag = "div"
return self.parser.tostring(element_body)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lefigaro/pages/flashactu.py 0000664 0000000 0000000 00000002417 11666415431 0031116 0 ustar 00root root 0000000 0000000 "ArticlePage object for inrocks"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage
class FlashActuPage(GenericNewsPage):
"ArticlePage object for inrocks"
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_title_selector = "h1"
self.element_author_selector = "div.name>span"
self.element_body_selector = "h2"
def get_body(self):
element_body = self.get_element_body()
element_body.tag = "div"
return self.parser.tostring(element_body)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lefigaro/pages/simple.py 0000664 0000000 0000000 00000002114 11666415431 0030427 0 ustar 00root root 0000000 0000000 "ArticlePage object for minutes20"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage
class SimplePage(GenericNewsPage):
"ArticlePage object for minutes20"
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_author_selector = "div.mna-signature"
self.element_body_selector = "#article"
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lefigaro/pages/special.py 0000664 0000000 0000000 00000002536 11666415431 0030566 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage, try_remove_from_selector_list
class SpecialPage(GenericNewsPage):
"ArticlePage object for inrocks"
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_title_selector = "h2"
self.element_author_selector = "div.name>span"
self.element_body_selector = ".block-text"
def get_body(self):
element_body = self.get_element_body()
try_remove_from_selector_list(self.parser, element_body, ['div'])
element_body.tag = "div"
return self.parser.tostring(element_body)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lefigaro/test.py 0000664 0000000 0000000 00000001723 11666415431 0027023 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
__all__ = ['LeFigaroTest']
class LeFigaroTest(BackendTest):
BACKEND = 'lefigaro'
def test_new_messages(self):
for message in self.backend.iter_unread_messages():
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lefigaro/tools.py 0000664 0000000 0000000 00000002417 11666415431 0027205 0 ustar 00root root 0000000 0000000 "tools for lefigaro backend"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
def id2url(_id):
"return an url from an id"
regexp2 = re.compile("(\w+).([0-9]+).(.*$)")
match = regexp2.match(_id)
if match:
return 'http://www.20minutes.fr/%s/%s/%s' % ( match.group(1),
match.group(2),
match.group(3))
else:
raise ValueError("id doesn't match")
def url2id(url):
"return an id from an url"
return url
def rssid(entry):
return url2id(entry.id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lemouv/ 0000775 0000000 0000000 00000000000 11666415431 0025206 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lemouv/__init__.py 0000664 0000000 0000000 00000001440 11666415431 0027316 0 ustar 00root root 0000000 0000000 # * -*- coding: utf-8 -*-
# Copyright(C) 2011 Johann Broudin
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import LeMouvBackend
__all__ = ['LeMouvBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lemouv/backend.py 0000664 0000000 0000000 00000006643 11666415431 0027160 0 ustar 00root root 0000000 0000000 # * -*- coding: utf-8 -*-
# Copyright(C) 2011 Johann Broudin
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.radio import ICapRadio, Radio, Stream, Emission
from weboob.capabilities.collection import ICapCollection, CollectionNotFound
from weboob.tools.backend import BaseBackend
from weboob.tools.browser import BaseBrowser, BasePage
__all__ = ['LeMouvBackend']
class XMLinfos(BasePage):
def get_current(self):
try:
for channel in self.parser.select(self.document.getroot(), 'channel'):
title = channel.find('item/song_title').text
artist = channel.find('item/artist_name').text
except AttributeError:
title = "Not defined"
artist = "Not defined"
return unicode(artist).strip(), unicode(title).strip()
class LeMouvBrowser(BaseBrowser):
DOMAIN = u'statique.lemouv.fr'
PAGES = {r'.*/files/rfPlayer/mouvRSS\.xml': XMLinfos}
def get_current(self, radio):
self.location('/files/rfPlayer/mouvRSS.xml')
assert self.is_on_page(XMLinfos)
return self.page.get_current()
class LeMouvBackend(BaseBackend, ICapRadio, ICapCollection):
NAME = 'lemouv'
MAINTAINER = 'Johann Broudin'
EMAIL = 'johann.broudin@6-8.fr'
VERSION = '0.9.1'
DESCRIPTION = u'The le mouv\' french radio'
LICENCE = 'AGPLv3+'
BROWSER = LeMouvBrowser
_RADIOS = {'lemouv': (u'le mouv\'', u'le mouv', u'http://mp3.live.tv-radio.com/lemouv/all/lemouvhautdebit.mp3')}
def iter_resources(self, splited_path):
if len(splited_path) > 0:
raise CollectionNotFound()
for id in self._RADIOS.iterkeys():
yield self.get_radio(id)
def iter_radios_search(self, pattern):
for radio in self.iter_resources([]):
if pattern.lower() in radio.title.lower() or pattern.lower() in radio.description.lower():
yield radio
def get_radio(self, radio):
if not isinstance(radio, Radio):
radio = Radio(radio)
if not radio.id in self._RADIOS:
return None
title, description, url = self._RADIOS[radio.id]
radio.title = title
radio.description = description
artist, title = self.browser.get_current(radio.id)
current = Emission(0)
current.artist = artist
current.title = title
radio.current = current
stream = Stream(0)
stream.title = u'128kbits/s'
stream.url = url
radio.streams = [stream]
return radio
def fill_radio(self, radio, fields):
if 'current' in fields:
if not radio.current:
radio.current = Emission(0)
radio.current.artist, radio.current.title = self.browser.get_current(radio.id)
return radio
OBJECTS = {Radio: fill_radio}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/lemouv/test.py 0000664 0000000 0000000 00000001663 11666415431 0026545 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class LeMouvTest(BackendTest):
BACKEND = 'lemouv'
def test_lemouv(self):
l = list(self.backend.iter_resources([]))
self.assertTrue(len(l) > 0)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangafox/ 0000775 0000000 0000000 00000000000 11666415431 0025477 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangafox/__init__.py 0000664 0000000 0000000 00000001440 11666415431 0027607 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import MangafoxBackend
__all__ = ['MangafoxBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangafox/backend.py 0000664 0000000 0000000 00000002646 11666415431 0027450 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderBackend, DisplayPage
__all__ = ['MangafoxBackend']
class MangafoxBackend(GenericComicReaderBackend):
NAME = 'mangafox'
DESCRIPTION = 'Mangafox manga reading site'
BROWSER_PARAMS = dict(
img_src_xpath="//img[@id='image']/attribute::src",
page_list_xpath="(//select[@onchange='change_page(this)'])[1]/option/@value",
page_to_location="%s.html")
ID_REGEXP = r'[^/]+/[^/]+(?:/[^/]+)?'
URL_REGEXP = r'.+mangafox.com/manga/(%s).+' % ID_REGEXP
ID_TO_URL = 'http://www.mangafox.com/manga/%s'
PAGES = { r'http://.+\.mangafox.\w+/manga/[^/]+/[^/]+/([^/]+/)?.+\.html': DisplayPage }
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangafox/test.py 0000664 0000000 0000000 00000001726 11666415431 0027036 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderTest
class MangafoxTest(GenericComicReaderTest):
BACKEND = 'mangafox'
def test_download(self):
return self._test_download('glass_no_kamen/v02/c000')
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangahere/ 0000775 0000000 0000000 00000000000 11666415431 0025626 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangahere/__init__.py 0000664 0000000 0000000 00000001442 11666415431 0027740 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import MangahereBackend
__all__ = ['MangahereBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangahere/backend.py 0000664 0000000 0000000 00000002546 11666415431 0027576 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderBackend, DisplayPage
__all__ = ['MangahereBackend']
class MangahereBackend(GenericComicReaderBackend):
NAME = 'mangahere'
DESCRIPTION = 'Mangahere manga reading site'
DOMAIN = 'www.mangahere.com'
BROWSER_PARAMS = dict(
img_src_xpath="//img[@id='image']/@src",
page_list_xpath="(//select[@onchange='change_page(this)'])[1]/option/@value")
ID_REGEXP = r'[^/]+/[^/]+/[^/]+'
URL_REGEXP = r'.+mangahere.com/manga/(%s).+' % ID_REGEXP
ID_TO_URL = 'http://www.mangahere.com/manga/%s'
PAGES = { URL_REGEXP: DisplayPage }
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangahere/test.py 0000664 0000000 0000000 00000001730 11666415431 0027160 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderTest
class MangahereTest(GenericComicReaderTest):
BACKEND = 'mangahere'
def test_download(self):
return self._test_download('glass_no_kamen/v02/c000')
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangareader/ 0000775 0000000 0000000 00000000000 11666415431 0026145 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangareader/__init__.py 0000664 0000000 0000000 00000001446 11666415431 0030263 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import MangareaderBackend
__all__ = ['MangareaderBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangareader/backend.py 0000664 0000000 0000000 00000002554 11666415431 0030114 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderBackend, DisplayPage
__all__ = ['MangareaderBackend']
class MangareaderBackend(GenericComicReaderBackend):
NAME = 'mangareader'
DESCRIPTION = 'Mangareader manga reading site'
DOMAIN = 'www.mangareader.net'
BROWSER_PARAMS = dict(
img_src_xpath="//img[@id='img']/@src",
page_list_xpath="//select[@id='pageMenu']/option/@value")
ID_REGEXP = r'[^/]+/[^/]+'
URL_REGEXP = r'.+mangareader.net/(%s).+' % ID_REGEXP
ID_TO_URL = 'http://www.mangareader.net/%s'
PAGES = { r'http://.+\.mangareader.net/.+': DisplayPage } # oh well
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangareader/test.py 0000664 0000000 0000000 00000001721 11666415431 0027477 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderTest
class MangareaderTest(GenericComicReaderTest):
BACKEND = 'mangareader'
def test_download(self):
return self._test_download('glass-mask/3')
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangatoshokan/ 0000775 0000000 0000000 00000000000 11666415431 0026531 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangatoshokan/__init__.py 0000664 0000000 0000000 00000001452 11666415431 0030644 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import MangatoshokanBackend
__all__ = ['MangatoshokanBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangatoshokan/backend.py 0000664 0000000 0000000 00000002625 11666415431 0030477 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderBackend, DisplayPage
__all__ = ['MangatoshokanBackend']
class MangatoshokanBackend(GenericComicReaderBackend):
NAME = 'mangatoshokan'
DESCRIPTION = 'Mangatoshokan manga reading site'
DOMAIN = "www.mangatoshokan.com"
BROWSER_PARAMS = dict(
img_src_xpath="//img[@id='readerPage']/@src",
page_list_xpath="(//select[@class='headerSelect'])[1]/option/@value")
ID_TO_URL = 'http://www.mangatoshokan.com/read/%s'
ID_REGEXP = r'[^/]+(?:/[^/]+)*'
URL_REGEXP = r'.+mangatoshokan.com/read/(%s)' % ID_REGEXP
PAGES = { r'http://.+mangatoshokan.com/read/.+': DisplayPage }
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mangatoshokan/test.py 0000664 0000000 0000000 00000001735 11666415431 0030070 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderTest
class MangatoshokanTest(GenericComicReaderTest):
BACKEND = 'mangatoshokan'
def test_download(self):
return self._test_download('Okujyouhime/Extras/5')
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mediawiki/ 0000775 0000000 0000000 00000000000 11666415431 0025642 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mediawiki/__init__.py 0000664 0000000 0000000 00000001446 11666415431 0027760 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Clément Schreiner
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import MediawikiBackend
__all__ = ['MediawikiBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mediawiki/backend.py 0000664 0000000 0000000 00000005312 11666415431 0027604 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Clément Schreiner
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.capabilities.content import ICapContent, Content
from weboob.tools.value import ValueBackendPassword, Value
from .browser import MediawikiBrowser
__all__ = ['MediawikiBackend']
class MediawikiBackend(BaseBackend, ICapContent):
NAME = 'mediawiki'
MAINTAINER = u'Clément Schreiner'
EMAIL = 'clemux@clemux.info'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = 'Mediawiki wiki software application'
CONFIG = BackendConfig(Value('url', label='URL of the Mediawiki website', default='http://en.wikipedia.org/'),
Value('apiurl', label='URL of the Mediawiki website\'s API', default='http://en.wikipedia.org/w/api.php'),
Value('username', label='Login', default=''),
ValueBackendPassword('password', label='Password', default=''))
BROWSER = MediawikiBrowser
def create_default_browser(self):
username = self.config['username'].get()
if len(username) > 0:
password = self.config['password'].get()
else:
password = None
return self.create_browser(self.config['url'].get(),
self.config['apiurl'].get(),
username, password)
def get_content(self, _id):
_id = _id.replace(' ', '_').encode('utf-8')
content = Content(_id)
page = _id
with self.browser:
data = self.browser.get_wiki_source(page)
content.content = data
return content
def iter_revisions(self, _id, max_results=10):
for rev in self.browser.iter_wiki_revisions(_id, max_results):
yield rev
def push_content(self, content, message=None, minor=False):
self.browser.set_wiki_source(content, message, minor)
def get_content_preview(self, content):
return self.browser.get_wiki_preview(content)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mediawiki/browser.py 0000664 0000000 0000000 00000014611 11666415431 0027702 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Clément Schreiner
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from urlparse import urlsplit
import urllib
import datetime
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from weboob.capabilities.content import Revision
try:
import simplejson
except ImportError:
# Python 2.6+ has a module similar to simplejson
import json as simplejson
__all__ = ['MediawikiBrowser']
class APIError(Exception):
pass
# Browser
class MediawikiBrowser(BaseBrowser):
ENCODING = 'utf-8'
def __init__(self, url, apiurl, *args, **kwargs):
url_parsed = urlsplit(url)
self.PROTOCOL = url_parsed.scheme
self.DOMAIN = url_parsed.netloc
self.BASEPATH = url_parsed.path
if self.BASEPATH.endswith('/'):
self.BASEPATH = self.BASEPATH[:-1]
self.apiurl = apiurl
BaseBrowser.__init__(self, *args, **kwargs)
def get_wiki_source(self, page):
assert isinstance(self.apiurl, basestring)
data = {'action': 'query',
'prop': 'revisions|info',
'titles': page,
'rvprop': 'content|timestamp',
'rvlimit': '1',
'intoken': 'edit',
}
result = self.API_get(data)
pageid = result['query']['pages'].keys()[0]
if pageid == "-1": # Page does not exist
return ""
return result['query']['pages'][str(pageid)]['revisions'][0]['*']
def get_token(self, page, _type):
''' _type can be edit, delete, protect, move, block, unblock, email or import'''
if len(self.username) > 0 and not self.is_logged():
self.login()
data = {'action': 'query',
'prop': 'info',
'titles': page,
'intoken': _type,
}
result = self.API_get(data)
pageid = result['query']['pages'].keys()[0]
return result['query']['pages'][str(pageid)][_type+'token']
def set_wiki_source(self, content, message=None, minor=False):
if len(self.username) > 0 and not self.is_logged():
self.login()
page = content.id
token = self.get_token(page, 'edit')
data = {'action': 'edit',
'title': page,
'token': token,
'text': content.content.encode('utf-8'),
'summary': message,
}
if minor:
data['minor'] = 'true'
self.API_post(data)
def get_wiki_preview(self, content, message=None):
data = {'action': 'parse',
'title': content.id,
'text': content.content.encode('utf-8'),
'summary': message,
}
result = self.API_post(data)
return result['parse']['text']['*']
def is_logged(self):
data = {'action': 'query',
'meta': 'userinfo',
}
result = self.API_get(data)
return result['query']['userinfo']['id'] != 0
def login(self):
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
assert isinstance(self.apiurl, basestring)
data = {'action': 'login',
'lgname': self.username,
'lgpassword': self.password,
}
result = self.API_post(data)
if result['login']['result'] == 'WrongPass':
raise BrowserIncorrectPassword()
if result['login']['result'] == 'NeedToken':
data['lgtoken'] = result['login']['token']
self.API_post(data)
def iter_wiki_revisions(self, page, nb_entries):
'''Yield 'Revision' objects for the last revisions of the specified page.'''
if len(self.username) > 0 and not self.is_logged():
self.login()
data = {'action': 'query',
'titles': page,
'prop': 'revisions',
'rvprop': 'ids|timestamp|comment|user|flags',
'rvlimit': str(nb_entries),
}
result = self.API_get(data)
pageid = str(result['query']['pages'].keys()[0])
if pageid != "-1":
for rev in result['query']['pages'][pageid]['revisions']:
rev_content = Revision(str(rev['revid']))
rev_content.comment = rev['comment']
rev_content.revision = str(rev['revid'])
rev_content.author = rev['user']
rev_content.timestamp = datetime.datetime.strptime(rev['timestamp'], '%Y-%m-%dT%H:%M:%SZ')
if rev.has_key('minor'):
rev_content.minor = True
else:
rev_content.minor = False
yield rev_content
def home(self):
'''We don't need to change location, we're using the JSON API here.'''
pass
def check_result(self, result):
if 'error' in result:
raise APIError('%s' % result['error']['info'])
def API_get(self, data):
'''Submit a GET request to the website
The JSON data is parsed and returned as a dictionary'''
data['format'] = 'json'
result = simplejson.loads(self.readurl(self.buildurl(self.apiurl, **data)), 'utf-8')
self.check_result(result)
return result
def API_post(self, data):
'''Submit a POST request to the website
The JSON data is parsed and returned as a dictionary'''
data['format'] = 'json'
result = simplejson.loads(self.readurl(self.apiurl, urllib.urlencode(data)), 'utf-8')
self.check_result(result)
return result
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/mediawiki/test.py 0000664 0000000 0000000 00000003121 11666415431 0027170 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Clément Schreiner
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
from datetime import datetime
class MediawikiTest(BackendTest):
BACKEND = 'mediawiki'
def test_get_content(self):
self.backend.get_content(u"Utilisateur:Clemux/Test")
def test_iter_revisions(self):
for rev in self.backend.iter_revisions(u"Utilisateur:Clemux/Test"):
pass
def test_push_content(self):
content = self.backend.get_content(u"Utilisateur:Clemux/Test")
content.content = "test "+str(datetime.now())
self.backend.push_content(content, message="test weboob", minor=True)
new_content = self.backend.get_content(u"Utilisateur:Clemux/Test")
assert content.content == new_content.content
def test_content_preview(self):
content = self.backend.get_content(u"Utilisateur:Clemux/Test")
self.backend.get_content_preview(content)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/meteofrance/ 0000775 0000000 0000000 00000000000 11666415431 0026167 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/meteofrance/__init__.py 0000664 0000000 0000000 00000001452 11666415431 0030302 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import MeteofranceBackend
__all__ = ['MeteofranceBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/meteofrance/backend.py 0000664 0000000 0000000 00000003010 11666415431 0030122 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Cedric Defortis
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.weather import ICapWeather
from weboob.tools.backend import BaseBackend
from .browser import MeteofranceBrowser
__all__ = ['MeteofranceBackend']
class MeteofranceBackend(BaseBackend, ICapWeather):
NAME = 'meteofrance'
MAINTAINER = 'Cedric Defortis'
EMAIL = 'cedric@aiur.fr'
VERSION = '0.9.1'
DESCRIPTION = 'Get forecasts from the MeteoFrance website'
LICENSE = 'AGPLv3+'
BROWSER = MeteofranceBrowser
def create_default_browser(self):
return self.create_browser()
def get_current(self, city_id):
return self.browser.get_current(city_id)
def iter_forecast(self, city_id):
return self.browser.iter_forecast(city_id)
def iter_city_search(self, pattern):
return self.browser.iter_city_search(pattern)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/meteofrance/browser.py 0000664 0000000 0000000 00000005122 11666415431 0030224 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Cedric Defortis
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import urllib
from weboob.tools.browser import BaseBrowser
from .pages.meteo import WeatherPage, CityPage
__all__ = ['MeteofranceBrowser']
class MeteofranceBrowser(BaseBrowser):
DOMAIN = 'france.meteofrance.com'
PROTOCOL = 'http'
ENCODING = 'utf-8'
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
WEATHER_URL = '{0}://{1}/france/meteo?PREVISIONS_PORTLET.path=previsionsville/{{cityid}}'.format(PROTOCOL, DOMAIN)
CITY_SEARCH_URL = '{0}://{1}/france/accueil/resultat?RECHERCHE_RESULTAT_PORTLET.path=rechercheresultat&' \
'query={{city_pattern}}&type=PREV_FRANCE&satellite=france'.format(PROTOCOL, DOMAIN)
PAGES = {
WEATHER_URL.format(cityid=".*"): WeatherPage,
CITY_SEARCH_URL.format(city_pattern=".*"): CityPage,
'http://france.meteofrance.com/france/accueil/resultat.*': CityPage,
'http://france.meteofrance.com/france/meteo.*': WeatherPage,
}
def __init__(self, *args, **kwargs):
BaseBrowser.__init__(self, *args, **kwargs)
def iter_city_search(self, pattern):
searchurl = self.CITY_SEARCH_URL.format(city_pattern=urllib.quote_plus(pattern.encode('utf-8')))
self.location(searchurl)
if self.is_on_page(CityPage):
# Case 1: there are multiple results for the pattern:
return self.page.iter_city_search()
else:
# Case 2: there is only one result, and the website send directly
# the browser on the forecast page:
return [self.page.get_city()]
def iter_forecast(self, city_id):
self.location(self.WEATHER_URL.format(cityid=city_id))
assert self.is_on_page(WeatherPage)
return self.page.iter_forecast()
def get_current(self, city_id):
self.location(self.WEATHER_URL.format(cityid=city_id))
assert self.is_on_page(WeatherPage)
return self.page.get_current()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/meteofrance/pages/ 0000775 0000000 0000000 00000000000 11666415431 0027266 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/meteofrance/pages/__init__.py 0000664 0000000 0000000 00000001336 11666415431 0031402 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/meteofrance/pages/meteo.py 0000664 0000000 0000000 00000007050 11666415431 0030753 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Cedric Defortis
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage
from weboob.capabilities.weather import Forecast, Current, City
import datetime
__all__ = ['WeatherPage', 'CityPage']
class WeatherPage(BasePage):
def get_temp_without_unit(self, temp_str):
# It seems that the mechanize module give us some old style
# ISO character
return int(temp_str.replace(u"\xb0C", "").strip())
def iter_forecast(self):
for div in self.document.getiterator('li'):
if div.attrib.get('class', '').startswith('jour'):
mdate = div.xpath('./dl/dt')[0].text
t_low = self.get_temp_without_unit(div.xpath('.//dd[@class="minmax"]/strong')[0].text)
t_high = self.get_temp_without_unit(div.xpath('.//dd[@class="minmax"]/strong')[1].text)
mtxt = div.xpath('.//dd')[0].text
yield Forecast(mdate, t_low, t_high, mtxt, 'C')
elif div.attrib.get('class', '').startswith('lijourle'):
for em in div.getiterator('em'):
templist = em.text_content().split("/")
t_low = self.get_temp_without_unit(templist[0])
t_high = self.get_temp_without_unit(templist[1])
break
for strong in div.getiterator("strong"):
mdate = strong.text_content()
break
for img in div.getiterator("img"):
mtxt = img.attrib["title"]
break
yield Forecast(mdate, t_low, t_high, mtxt, "C")
def get_current(self):
div = self.document.getroot().xpath('//div[@class="bloc_details"]/ul/li/dl')[0]
mdate = datetime.datetime.now()
temp = self.get_temp_without_unit(div.xpath('./dd[@class="minmax"]')[0].text)
mtxt = div.find('dd').find('img').attrib['title']
return Current(mdate, temp, mtxt, 'C')
def get_city(self):
"""
Return the city from the forecastpage.
"""
for div in self.document.getiterator('div'):
if div.attrib.has_key("class") and div.attrib.get("class") == "choix":
for strong in div.getiterator("strong"):
city_name=strong.text +" "+ strong.tail.replace("(","").replace(")","")
city_id=self.url.split("/")[-1]
return City(city_id, city_name)
class CityPage(BasePage):
def iter_city_search(self):
for div in self.document.getiterator('div'):
if div.attrib.get('id') == "column1":
for li in div.getiterator('li'):
city_name = li.text_content()
for children in li.getchildren():
city_id = children.attrib.get("href").split("/")[-1]
mcity = City( city_id, city_name)
yield mcity
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/meteofrance/test.py 0000664 0000000 0000000 00000002277 11666415431 0027530 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class MeteoFranceTest(BackendTest):
BACKEND = 'meteofrance'
def test_meteofrance(self):
l = list(self.backend.iter_city_search('paris'))
self.assertTrue(len(l) > 0)
city = l[0]
current = self.backend.get_current(city.id)
self.assertTrue(current.temp > -20 and current.temp < 50)
forecasts = list(self.backend.iter_forecast(city.id))
self.assertTrue(len(forecasts) > 0)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/minutes20/ 0000775 0000000 0000000 00000000000 11666415431 0025525 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/minutes20/__init__.py 0000664 0000000 0000000 00000001526 11666415431 0027642 0 ustar 00root root 0000000 0000000 "init of Newspaper20minutesBackend"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import Newspaper20minutesBackend
__all__ = ['Newspaper20minutesBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/minutes20/backend.py 0000664 0000000 0000000 00000002606 11666415431 0027472 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
"backend for http://20minutes.fr"
from weboob.capabilities.messages import ICapMessages
from weboob.tools.capabilities.messages.GenericBackend import GenericNewspaperBackend
from .browser import Newspaper20minutesBrowser
from .tools import rssid
class Newspaper20minutesBackend(GenericNewspaperBackend, ICapMessages):
"Newspaper20minutesBackend class"
MAINTAINER = 'Julien Hebert'
EMAIL = 'juke@free.fr'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
STORAGE = {'seen': {}}
NAME = 'minutes20'
DESCRIPTION = u'20minutes French news website'
BROWSER = Newspaper20minutesBrowser
RSS_FEED = 'http://www.20minutes.fr/rss/20minutes.xml'
RSSID = rssid
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/minutes20/browser.py 0000664 0000000 0000000 00000003124 11666415431 0027562 0 ustar 00root root 0000000 0000000 "browser for 20minutes website"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .pages.article import ArticlePage
from .pages.simple import SimplePage
from weboob.tools.browser import BaseBrowser
from .tools import id2url
class Newspaper20minutesBrowser(BaseBrowser):
"Newspaper20minutesBrowser class"
PAGES = {
'http://www.20minutes.fr/article/?.*': ArticlePage,
'http://www.20minutes.fr/ledirect/?.*': SimplePage,
'http://www.20minutes.fr/preums/?.*': SimplePage
}
def is_logged(self):
return False
def login(self):
pass
def fillobj(self, obj, fields):
pass
def get_content(self, _id):
"return page article content"
try :
url = id2url(_id)
except ValueError:
url = _id
self.location(url)
if self.page is not None:
return self.page.get_article(_id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/minutes20/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026624 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/minutes20/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0030723 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/minutes20/pages/article.py 0000664 0000000 0000000 00000003227 11666415431 0030625 0 ustar 00root root 0000000 0000000 "ArticlePage object for minutes20"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.messages.genericArticle import NoAuthorElement, try_remove, NoneMainDiv
from .simple import SimplePage
class ArticlePage(SimplePage):
"ArticlePage object for minutes20"
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_title_selector = "h1"
self.element_author_selector = "div.mna-signature"
self.element_body_selector = "div.mna-body"
def get_body(self):
try:
element_body = self.get_element_body()
except NoneMainDiv:
return None
else:
try_remove(self.parser, element_body, "div.mna-tools")
try_remove(self.parser, element_body, "div.mna-comment-call")
try :
element_body.remove(self.get_element_author())
except NoAuthorElement:
pass
return self.parser.tostring(element_body)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/minutes20/pages/simple.py 0000664 0000000 0000000 00000002173 11666415431 0030472 0 ustar 00root root 0000000 0000000 "ArticlePage object for minutes20"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage
class SimplePage(GenericNewsPage):
"ArticlePage object for minutes20"
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_title_selector = "h1"
self.element_author_selector = "div.mna-signature"
self.element_body_selector = "div.mna-body"
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/minutes20/test.py 0000664 0000000 0000000 00000001726 11666415431 0027064 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
__all__ = ['Minutes20Test']
class Minutes20Test(BackendTest):
BACKEND = 'minutes20'
def test_new_messages(self):
for message in self.backend.iter_unread_messages():
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/minutes20/tools.py 0000664 0000000 0000000 00000002674 11666415431 0027250 0 ustar 00root root 0000000 0000000 "common tools for 20minutes backend"
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
def id2url(_id):
"return an url from an id"
regexp2 = re.compile("(\w+).([0-9]+).(.*$)")
match = regexp2.match(_id)
if match:
return 'http://www.20minutes.fr/%s/%s/%s' % ( match.group(1),
match.group(2),
match.group(3))
else:
raise ValueError("id doesn't match")
def url2id(url):
"return an id from an url"
regexp = re.compile("http://www.20minutes.fr/(\w+)/([0-9]+)/(.*$)")
match = regexp.match(url)
return '%s.%d.%s' % (match.group(1), int(match.group(2)), match.group(3))
def rssid(entry):
return url2id(entry.id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/newsfeed/ 0000775 0000000 0000000 00000000000 11666415431 0025477 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/newsfeed/__init__.py 0000664 0000000 0000000 00000001451 11666415431 0027611 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Clément Schreiner
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import NewsfeedBackend
__all__ = ['NewsfeedBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/newsfeed/backend.py 0000664 0000000 0000000 00000005761 11666415431 0027451 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Clément Schreiner
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.capabilities.messages import ICapMessages, Message, Thread
from weboob.tools.newsfeed import Newsfeed
from weboob.tools.value import Value
__all__ = ['NewsfeedBackend']
class NewsfeedBackend(BaseBackend, ICapMessages):
NAME = 'newsfeed'
MAINTAINER = u"Clément Schreiner"
EMAIL = "clemux@clemux.info"
VERSION = '0.9.1'
DESCRIPTION = "Loads RSS and Atom feeds from any website"
LICENSE = "AGPLv3+"
CONFIG = BackendConfig(Value('url', label="Atom/RSS feed's url"))
STORAGE = {'seen': []}
def iter_threads(self):
for article in Newsfeed(self.config['url'].get()).iter_entries():
yield self.get_thread(article.id, article)
def get_thread(self, id, entry=None):
if isinstance(id, Thread):
thread = id
id = thread.id
else:
thread = Thread(id)
if entry is None:
entry = Newsfeed(self.config['url'].get()).get_entry(id)
if entry is None:
return None
flags = Message.IS_HTML
if not thread.id in self.storage.get('seen', default=[]):
flags |= Message.IS_UNREAD
if len(entry.content):
content = entry.content[0]
else:
content = entry.link
thread.title = entry.title
thread.root = Message(thread=thread,
id=0,
title=entry.title,
sender=entry.author,
receivers=None,
date=entry.datetime,
parent=None,
content=content,
children=[],
flags=flags)
return thread
def iter_unread_messages(self, thread=None):
for thread in self.iter_threads():
for m in thread.iter_all_messages():
if m.flags & m.IS_UNREAD:
yield m
def set_message_read(self, message):
self.storage.get('seen', default=[]).append(message.thread.id)
self.storage.save()
def fill_thread(self, thread, fields):
return self.get_thread(thread)
OBJECTS = {Thread: fill_thread}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/newsfeed/test.py 0000664 0000000 0000000 00000001702 11666415431 0027030 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Clément Schreiner
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class NewsfeedTest(BackendTest):
BACKEND = 'newsfeed'
def test_newsfeed(self):
for message in self.backend.iter_unread_messages():
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/nova/ 0000775 0000000 0000000 00000000000 11666415431 0024642 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/nova/__init__.py 0000664 0000000 0000000 00000001426 11666415431 0026756 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import NovaBackend
__all__ = ['NovaBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/nova/backend.py 0000664 0000000 0000000 00000007133 11666415431 0026607 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.radio import ICapRadio, Radio, Stream, Emission
from weboob.capabilities.collection import ICapCollection, CollectionNotFound
from weboob.tools.backend import BaseBackend
from weboob.tools.browser import BaseBrowser, BasePage, BrowserUnavailable
__all__ = ['NovaBackend']
class HistoryPage(BasePage):
def on_loaded(self):
h2 = self.parser.select(self.document.getroot(), 'h2')
if len(h2) > 0 and h2[0].text == 'Site off-line':
raise BrowserUnavailable('Website is currently offline')
def get_current(self):
for div in self.parser.select(self.document.getroot(), 'div#rubrique_contenu div.resultat'):
artist = self.parser.select(div, 'span#artiste', 1).find('a').text
title = self.parser.select(div, 'span#titre', 1).text
return unicode(artist).strip(), unicode(title).strip()
class NovaBrowser(BaseBrowser):
DOMAIN = u'www.novaplanet.com'
PAGES = {r'http://www.novaplanet.com/cetaitquoicetitre/radionova/\d*': HistoryPage,
}
def get_current(self):
self.location('/cetaitquoicetitre/radionova/')
assert self.is_on_page(HistoryPage)
return self.page.get_current()
class NovaBackend(BaseBackend, ICapRadio, ICapCollection):
NAME = 'nova'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
DESCRIPTION = u'Nova french radio'
LICENSE = 'AGPLv3+'
BROWSER = NovaBrowser
_RADIOS = {'nova': (u'Radio Nova', u'Radio nova', u'http://broadcast.infomaniak.net:80/radionova-high.mp3'),
}
def iter_resources(self, splited_path):
if len(splited_path) > 0:
raise CollectionNotFound()
for id in self._RADIOS.iterkeys():
yield self.get_radio(id)
def iter_radios_search(self, pattern):
for radio in self.iter_resources([]):
if pattern.lower() in radio.title.lower() or pattern.lower() in radio.description.lower():
yield radio
def get_radio(self, radio):
if not isinstance(radio, Radio):
radio = Radio(radio)
if not radio.id in self._RADIOS:
return None
title, description, url = self._RADIOS[radio.id]
radio.title = title
radio.description = description
artist, title = self.browser.get_current()
current = Emission(0)
current.artist = artist
current.title = title
radio.current = current
stream = Stream(0)
stream.title = u'128kbits/s'
stream.url = url
radio.streams = [stream]
return radio
def fill_radio(self, radio, fields):
if 'current' in fields:
if not radio.current:
radio.current = Emission(0)
radio.current.artist, radio.current.title = self.browser.get_current()
return radio
OBJECTS = {Radio: fill_radio}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/nova/test.py 0000664 0000000 0000000 00000001650 11666415431 0026175 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class NovaTest(BackendTest):
BACKEND = 'nova'
def test_nova(self):
l = list(self.backend.iter_resources([]))
self.assertTrue(len(l) > 0)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/orange/ 0000775 0000000 0000000 00000000000 11666415431 0025152 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/orange/__init__.py 0000664 0000000 0000000 00000000100 11666415431 0027252 0 ustar 00root root 0000000 0000000 from .backend import OrangeBackend
__all__ = ['OrangeBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/orange/backend.py 0000664 0000000 0000000 00000004372 11666415431 0027121 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.messages import CantSendMessage, ICapMessages, ICapMessagesPost
from weboob.capabilities.account import ICapAccount, StatusField
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import ValueBackendPassword, Value
from .browser import OrangeBrowser
__all__ = ['OrangeBackend']
class OrangeBackend(BaseBackend, ICapAccount, ICapMessages, ICapMessagesPost):
NAME = 'orange'
MAINTAINER = 'Nicolas Duhamel'
EMAIL = 'nicolas@jombi.fr'
VERSION = '0.9.1'
DESCRIPTION = 'Orange french mobile phone provider'
LICENSE = 'AGPLv3+'
CONFIG = BackendConfig(Value('login', label='Login'),
ValueBackendPassword('password', label='Password'),
Value('phonenumber', Label='Phone number')
)
BROWSER = OrangeBrowser
ACCOUNT_REGISTER_PROPERTIES = None
def create_default_browser(self):
return self.create_browser(self.config['login'].get(), self.config['password'].get())
def get_account_status(self):
with self.browser:
return (StatusField('nb_remaining_free_sms', 'Number of remaining free SMS',
self.browser.get_nb_remaining_free_sms()),)
def post_message(self, message):
if not message.content.strip():
raise CantSendMessage(u'Message content is empty.')
with self.browser:
self.browser.post_message(message, self.config['phonenumber'].get())
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/orange/browser.py 0000664 0000000 0000000 00000004355 11666415431 0027216 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
#~ from .pages.compose import ClosePage, ComposePage, ConfirmPage, SentPage
#~ from .pages.login import LoginPage
from .pages import LoginPage, ComposePage, ConfirmPage
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
__all__ = ['OrangeBrowser']
class OrangeBrowser(BaseBrowser):
DOMAIN = 'orange.fr'
PAGES = {
'http://id.orange.fr/auth_user/bin/auth_user.cgi.*': LoginPage,
'http://id.orange.fr/auth_user/bin/auth0user.cgi.*': LoginPage,
'http://smsmms1.orange.fr/./Sms/sms_write.php.*' : ComposePage,
'http://smsmms1.orange.fr/./Sms/sms_write.php?command=send' : ConfirmPage,
}
def get_nb_remaining_free_sms(self):
self.location("http://smsmms1.orange.fr/M/Sms/sms_write.php")
return self.page.get_nb_remaining_free_sms()
def home(self):
self.location("http://smsmms1.orange.fr/M/Sms/sms_write.php")
def is_logged(self):
self.location("http://smsmms1.orange.fr/M/Sms/sms_write.php", no_login=True)
return not self.is_on_page(LoginPage)
def login(self):
if not self.is_on_page(LoginPage):
self.location('http://id.orange.fr/auth_user/bin/auth_user.cgi?url=http://www.orange.fr', no_login=True)
self.page.login(self.username, self.password)
if not self.is_logged():
raise BrowserIncorrectPassword()
def post_message(self, message, sender):
if not self.is_on_page(ComposePage):
self.home()
self.page.post_message(message, sender)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/orange/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026251 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/orange/pages/__init__.py 0000664 0000000 0000000 00000001543 11666415431 0030365 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .login import LoginPage
from .compose import ComposePage, ConfirmPage
__all__ = ['LoginPage', 'ComposePage', 'ConfirmPage']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/orange/pages/compose.py 0000664 0000000 0000000 00000004377 11666415431 0030303 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from weboob.capabilities.messages import CantSendMessage
from weboob.tools.browser import BasePage
__all__ = ['ComposePage', 'ConfirmPage']
class ConfirmPage(BasePage):
def on_loaded(self):
pass
class ComposePage(BasePage):
phone_regex = re.compile('^(\+33|0033|0)(6|7)(\d{8})$')
def on_loaded(self):
#Deal with bad encoding... for ie6 ...
response = self.browser.response()
response.set_data(response.get_data().decode('utf-8', 'ignore') )
self.browser.set_response(response)
def get_nb_remaining_free_sms(self):
return "0"
def post_message(self, message, sender):
receiver = message.thread.id
if self.phone_regex.match(receiver) is None:
raise CantSendMessage(u'Invalid receiver: %s' % receiver)
listetel = ",,"+ receiver
#Fill the form
self.browser.select_form(name="formulaire")
self.browser.new_control("hidden", "autorize",{'value':''})
self.browser.new_control("textarea", "msg", {'value':''})
self.browser.set_all_readonly(False)
self.browser["corpsms"] = message.content
self.browser["pays"] = "33"
self.browser["listetel"] = listetel
self.browser["reply"] = "2"
self.browser["typesms"] = "2"
self.browser["produit"] = "1000"
self.browser["destToKeep"] = listetel
self.browser["NUMTEL"] = sender
self.browser["autorize"] = "1"
self.browser["msg"] = message.content.encode('utf-8')
self.browser.submit()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/orange/pages/login.py 0000664 0000000 0000000 00000004100 11666415431 0027726 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage
import urllib
__all__ = ['LoginPage']
class LoginPage(BasePage):
def on_loaded(self):
pass
def login(self, user, pwd):
post_data = {"credential" : str(user),
"pwd" : str(pwd),
"save_user": "false",
"save_pwd" : "false",
"save_TC" : "true",
"action" : "valider",
"usertype" : "",
"service" : "",
"url" : "http://www.orange.fr",
"case" : "",
"origin" : "", }
post_data = urllib.urlencode(post_data)
self.browser.addheaders = [('Referer', 'http://id.orange.fr/auth_user/template/auth0user/htm/vide.html'),
("Content-Type" , 'application/x-www-form-urlencoded') ]
self.browser.open(self.browser.geturl(), data=post_data)
#~ print "LOGIN!!!"
#~ self.browser.select_form(predicate=lambda form: "id" in form.attrs and form.attrs["id"] == "authentication_form" )
#~ user_control = self.browser.find_control(id="user_credential")
#~ user_control.value = user
#~ pwd_control = self.browser.find_control(id="user_password")
#~ pwd_control.value = pwd
#~ self.browser.submit()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ouifm/ 0000775 0000000 0000000 00000000000 11666415431 0025016 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ouifm/__init__.py 0000664 0000000 0000000 00000001435 11666415431 0027132 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import OuiFMBackend
__all__ = ['OuiFMBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ouifm/backend.py 0000664 0000000 0000000 00000007215 11666415431 0026764 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.radio import ICapRadio, Radio, Stream, Emission
from weboob.capabilities.collection import ICapCollection, CollectionNotFound
from weboob.tools.backend import BaseBackend
from weboob.tools.browser import StandardBrowser
__all__ = ['OuiFMBackend']
class OuiFMBackend(BaseBackend, ICapRadio, ICapCollection):
NAME = 'ouifm'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
DESCRIPTION = u'The Ouï FM french radio'
LICENSE = 'AGPLv3+'
BROWSER = StandardBrowser
_RADIOS = {'general': (u'OUÏ FM', u'OUI FM', u'http://ouifm.ice.infomaniak.ch/ouifm-high.mp3'),
'alternatif': (u'OUÏ FM Alternatif', u'OUI FM - L\'Alternative Rock', u'http://ouifm.ice.infomaniak.ch/ouifm2.mp3'),
'collector': (u'OUÏ FM Collector', u'OUI FM - Classic Rock', u'http://ouifm.ice.infomaniak.ch/ouifm3.mp3'),
'blues': (u'OUÏ FM Blues', u'OUI FM - Blues', u'http://ouifm.ice.infomaniak.ch/ouifm4.mp3'),
'inde': (u'OUÏ FM Indé', u'OUI FM - Rock Indé', u'http://ouifm.ice.infomaniak.ch/ouifm5.mp3'),
}
def create_default_browser(self):
return self.create_browser(parser='json')
def iter_resources(self, splited_path):
if len(splited_path) > 0:
raise CollectionNotFound()
for id in self._RADIOS.iterkeys():
yield self.get_radio(id)
def iter_radios_search(self, pattern):
for radio in self.iter_resources([]):
if pattern.lower() in radio.title.lower() or pattern.lower() in radio.description.lower():
yield radio
def get_current(self, radio):
document = self.browser.location('http://rock.ouifm.fr/dynamic-menu.json')
suffix = ''
if radio != 'general':
suffix = '_%s' % radio
last = document['last%s' % suffix][0]
return last['artiste%s' % suffix].strip(), last['titre%s' % suffix].strip()
def get_radio(self, radio):
if not isinstance(radio, Radio):
radio = Radio(radio)
if not radio.id in self._RADIOS:
return None
title, description, url = self._RADIOS[radio.id]
radio.title = title
radio.description = description
artist, title = self.get_current(radio.id)
current = Emission(0)
current.artist = artist
current.title = title
radio.current = current
stream = Stream(0)
stream.title = u'128kbits/s'
stream.url = url
radio.streams = [stream]
return radio
def fill_radio(self, radio, fields):
if 'current' in fields:
if not radio.current:
radio.current = Emission(0)
radio.current.artist, radio.current.title = self.get_current(radio.id)
return radio
OBJECTS = {Radio: fill_radio}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/ouifm/test.py 0000664 0000000 0000000 00000001660 11666415431 0026352 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class OuiFMTest(BackendTest):
BACKEND = 'ouifm'
def test_ouifm(self):
l = list(self.backend.iter_resources([]))
self.assertTrue(len(l) > 0)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/pastealacon/ 0000775 0000000 0000000 00000000000 11666415431 0026171 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/pastealacon/__init__.py 0000664 0000000 0000000 00000001450 11666415431 0030302 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import PastealaconBackend
__all__ = ['PastealaconBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/pastealacon/backend.py 0000664 0000000 0000000 00000005646 11666415431 0030145 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
import re
from weboob.tools.capabilities.paste import BasePasteBackend
from weboob.tools.backend import BaseBackend
from weboob.capabilities.base import NotLoaded
from .browser import PastealaconBrowser
from .paste import PastealaconPaste
__all__ = ['PastealaconBackend']
class PastealaconBackend(BaseBackend, BasePasteBackend):
NAME = 'pastealacon'
MAINTAINER = 'Laurent Bachelier'
EMAIL = 'laurent@bachelier.name'
VERSION = '0.9.1'
DESCRIPTION = 'Paste a la con paste tool'
LICENSE = 'AGPLv3+'
BROWSER = PastealaconBrowser
EXPIRATIONS = {
24*3600: 'd',
24*3600*30: 'm',
False: 'f',
}
def new_paste(self, *args, **kwargs):
return PastealaconPaste(*args, **kwargs)
def can_post(self, contents, title=None, public=None, max_age=None):
try:
contents.encode(self.browser.ENCODING)
except UnicodeEncodeError:
return 0
if public is False:
return 0
if max_age is not None:
if self.get_closest_expiration(max_age) is None:
return 0
# the "title" is filtered (does not even accepts dots)
if not title or re.match('^\w+$', title) and len(title) <= 24:
return 2
return 1
def get_paste(self, _id):
with self.browser:
return self.browser.get_paste(_id)
def fill_paste(self, paste, fields):
# if we only want the contents
if fields == ['contents']:
if paste.contents is NotLoaded:
with self.browser:
contents = self.browser.get_contents(paste.id)
paste.contents = contents
# get all fields
elif fields is None or len(fields):
with self.browser:
self.browser.fill_paste(paste)
return paste
def post_paste(self, paste, max_age = None):
if max_age is not None:
expiration = self.get_closest_expiration(max_age)
else:
expiration = None
with self.browser:
self.browser.post_paste(paste, expiration=self.EXPIRATIONS.get(expiration))
OBJECTS = {PastealaconPaste: fill_paste}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/pastealacon/browser.py 0000664 0000000 0000000 00000005154 11666415431 0030233 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from mechanize import RobustFactory
import re
from weboob.tools.browser import BaseBrowser, BrowserUnavailable, BrowserHTTPNotFound
from weboob.capabilities.paste import PasteNotFound
from weboob.tools.browser.decorators import id2url, check_url
from .pages import PastePage, CaptchaPage, PostPage
from .paste import PastealaconPaste
__all__ = ['PastealaconBrowser']
class PastealaconBrowser(BaseBrowser):
DOMAIN = 'pastealacon.com'
ENCODING = 'ISO-8859-1'
PASTE_URL = 'http://%s/(?P\d+)' % DOMAIN
PAGES = {PASTE_URL: PastePage,
'http://%s/%s' % (DOMAIN, re.escape('pastebin.php?captcha=1')): CaptchaPage,
'http://%s/' % DOMAIN: PostPage}
def __init__(self, *args, **kwargs):
kwargs['factory'] = RobustFactory()
BaseBrowser.__init__(self, *args, **kwargs)
@id2url(PastealaconPaste.id2url)
@check_url(PASTE_URL)
def get_paste(self, url):
_id = re.match('^%s$' % self.PASTE_URL, url).groupdict()['id']
return PastealaconPaste(_id)
def fill_paste(self, paste):
"""
Get as much as information possible from the paste page
"""
self.location(paste.page_url)
return self.page.fill_paste(paste)
def get_contents(self, _id):
"""
Get the contents from the raw URL
This is the fastest and safest method if you only want the content.
Returns unicode.
"""
try:
return self.readurl('http://%s/pastebin.php?dl=%s' % (self.DOMAIN, _id), if_fail='raise').decode(self.ENCODING)
except BrowserHTTPNotFound:
raise PasteNotFound()
def post_paste(self, paste, expiration=None):
self.home()
self.page.post(paste, expiration=expiration)
if self.is_on_page(CaptchaPage):
raise BrowserUnavailable("Detected as spam and unable to handle the captcha")
paste.id = self.page.get_id()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/pastealacon/pages.py 0000664 0000000 0000000 00000004110 11666415431 0027636 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from weboob.tools.browser import BasePage, BrokenPageError
from weboob.capabilities.paste import PasteNotFound
__all__ = ['PastePage', 'PostPage', 'CaptchaPage']
class PastePage(BasePage):
def fill_paste(self, paste):
root = self.document.getroot()
try:
# there is no 404, try to detect if there really is a content
self.parser.select(root, 'id("content")/div[@class="syntax"]//ol', 1, 'xpath')
except BrokenPageError:
raise PasteNotFound()
header = self.parser.select(root, 'id("content")/h3', 1, 'xpath')
matches = re.match(r'Posted by (?P.+) on (?P.+) \(', header.text)
paste.title = matches.groupdict().get('author')
paste.contents = self.parser.select(root, '//textarea[@id="code"]', 1, 'xpath').text
return paste
def get_id(self):
"""
Find out the ID from the URL
"""
return self.group_dict['id']
class PostPage(BasePage):
def post(self, paste, expiration=None):
self.browser.select_form(name='editor')
self.browser['code2'] = paste.contents.encode(self.browser.ENCODING)
self.browser['poster'] = paste.title.encode(self.browser.ENCODING)
if expiration:
self.browser['expiry'] = [expiration]
self.browser.submit()
class CaptchaPage(BasePage):
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/pastealacon/paste.py 0000664 0000000 0000000 00000001740 11666415431 0027661 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.paste import BasePaste
__all__ = ['PastealaconPaste']
class PastealaconPaste(BasePaste):
# all pastes are public
public = True
@classmethod
def id2url(cls, _id):
return 'http://pastealacon.com/%s' % _id
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/pastealacon/test.py 0000664 0000000 0000000 00000007503 11666415431 0027527 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
from weboob.capabilities.base import NotLoaded
from weboob.tools.browser import BrowserUnavailable
from weboob.capabilities.paste import PasteNotFound
class PastealaconTest(BackendTest):
BACKEND = 'pastealacon'
def _get_paste(self, _id):
# html method
p = self.backend.get_paste(_id)
self.backend.fillobj(p, ['title'])
assert p.title == 'ouiboube'
assert p.page_url.startswith('http://pastealacon.com/')
assert u'héhéhé' in p.contents
assert p.public is True
# raw method
p = self.backend.get_paste(_id)
self.backend.fillobj(p, ['contents'])
assert p.title is NotLoaded
assert p.page_url.startswith('http://pastealacon.com/')
assert u'héhéhé' in p.contents
assert p.public is True
def test_post(self):
p = self.backend.new_paste(None, title='ouiboube', contents=u'Weboob Test héhéhé')
self.backend.post_paste(p, max_age=3600*24)
assert p.id
self.backend.fill_paste(p, ['title'])
assert p.title == 'ouiboube'
assert p.id in p.page_url
assert p.public is True
# test all get methods from the Paste we just created
self._get_paste(p.id)
# same but from the full URL
self._get_paste('http://pastealacon.com/'+p.id)
def test_spam(self):
p = self.backend.new_paste(None, title='viagra', contents='http://example.com/')
self.assertRaises(BrowserUnavailable, self.backend.post_paste, p)
def test_notfound(self):
for _id in ('424242424242424242424242424242424242', 'http://pastealacon.com/424242424242424242424242424242424242'):
# html method
p = self.backend.get_paste(_id)
self.assertRaises(PasteNotFound, self.backend.fillobj, p, ['title'])
# raw method
p = self.backend.get_paste(_id)
self.assertRaises(PasteNotFound, self.backend.fillobj, p, ['contents'])
def test_checkurl(self):
# call with an URL we can't handle with this backend
assert self.backend.get_paste('http://pastebin.com/nJG9ZFG8') is None
# same even with correct domain (IDs are numeric)
assert self.backend.get_paste('http://pastealacon.com/nJG9ZFG8') is None
assert self.backend.get_paste('nJG9ZFG8') is None
def test_can_post(self):
assert 0 == self.backend.can_post('hello', public=False)
assert 1 <= self.backend.can_post('hello', public=True)
assert 0 == self.backend.can_post('hello', public=True, max_age=600)
assert 1 <= self.backend.can_post('hello', public=True, max_age=3600*24)
assert 1 <= self.backend.can_post('hello', public=True, max_age=3600*24*3)
assert 1 <= self.backend.can_post('hello', public=True, max_age=False)
assert 1 <= self.backend.can_post('hello', public=None, max_age=False)
assert 1 <= self.backend.can_post('hello', public=True, max_age=3600*24*40)
assert 1 <= self.backend.can_post(u'héhé', public=True)
assert 0 == self.backend.can_post(u'hello ♥', public=True)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/pastebin/ 0000775 0000000 0000000 00000000000 11666415431 0025504 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/pastebin/__init__.py 0000664 0000000 0000000 00000001442 11666415431 0027616 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import PastebinBackend
__all__ = ['PastebinBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/pastebin/backend.py 0000664 0000000 0000000 00000007072 11666415431 0027453 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.tools.capabilities.paste import BasePasteBackend
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.capabilities.base import NotLoaded
from weboob.tools.value import Value, ValueBackendPassword
from .browser import PastebinBrowser
from .paste import PastebinPaste
__all__ = ['PastebinBackend']
class PastebinBackend(BaseBackend, BasePasteBackend):
NAME = 'pastebin'
MAINTAINER = 'Laurent Bachelier'
EMAIL = 'laurent@bachelier.name'
VERSION = '0.9.1'
DESCRIPTION = 'Pastebin paste tool'
LICENSE = 'AGPLv3+'
BROWSER = PastebinBrowser
CONFIG = BackendConfig(
Value('username', label='Optional username', default=''),
ValueBackendPassword('password', label='Optional password', default=''),
ValueBackendPassword('api_key', label='Optional API key', default='', noprompt=True),
)
EXPIRATIONS = {
600: '10M',
3600: '1H',
3600*24: '1D',
3600*24*30: '1M',
False: 'N',
}
def create_default_browser(self):
username = self.config['username'].get()
if username:
password = self.config['password'].get()
else:
password = None
return self.create_browser(self.config['api_key'].get() if self.config['api_key'].get() else None,
username, password, get_home=False)
def new_paste(self, *args, **kwargs):
return PastebinPaste(*args, **kwargs)
def can_post(self, contents, title=None, public=None, max_age=None):
if max_age is not None:
if self.get_closest_expiration(max_age) is None:
return 0
if not title or len(title) <= 60:
return 2
return 1
def get_paste(self, _id):
with self.browser:
return self.browser.get_paste(_id)
def fill_paste(self, paste, fields):
# if we only want the contents
if fields == ['contents']:
if paste.contents is NotLoaded:
with self.browser:
contents = self.browser.get_contents(paste.id)
paste.contents = contents
# get all fields
elif fields is None or len(fields):
with self.browser:
self.browser.fill_paste(paste)
return paste
def post_paste(self, paste, max_age=None, use_api=True):
if max_age is not None:
expiration = self.get_closest_expiration(max_age)
else:
expiration = None
with self.browser:
if use_api and self.config.get('api_key').get():
self.browser.api_post_paste(paste, expiration=self.EXPIRATIONS.get(expiration))
else:
self.browser.post_paste(paste, expiration=self.EXPIRATIONS.get(expiration))
OBJECTS = {PastebinPaste: fill_paste}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/pastebin/browser.py 0000664 0000000 0000000 00000011175 11666415431 0027546 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser, BrowserHTTPNotFound, BrowserIncorrectPassword
from weboob.tools.browser.decorators import id2url, check_url
from weboob.capabilities.paste import PasteNotFound
from .pages import PastePage, PostPage
from .paste import PastebinPaste
import urllib
import re
__all__ = ['PastebinBrowser']
class BadAPIRequest(Exception):
pass
class PastebinBrowser(BaseBrowser):
DOMAIN = 'pastebin.com'
ENCODING = 'UTF-8'
PASTE_URL = 'http://%s/(?P\w+)' % DOMAIN
API_URL = 'http://%s/api/api_post.php' % DOMAIN
PAGES = {PASTE_URL: PastePage,
'http://%s/' % DOMAIN: PostPage}
def __init__(self, api_key, *args, **kwargs):
self.api_key = api_key
self.user_key = None
BaseBrowser.__init__(self, *args, **kwargs)
def fill_paste(self, paste):
"""
Get as much as information possible from the paste page
"""
try:
self.location(paste.page_url, no_login=True)
return self.page.fill_paste(paste)
except BrowserHTTPNotFound:
raise PasteNotFound()
@id2url(PastebinPaste.id2url)
@check_url(PASTE_URL)
def get_paste(self, url):
_id = re.match('^%s$' % self.PASTE_URL, url).groupdict()['id']
return PastebinPaste(_id)
def get_contents(self, _id):
"""
Get the contents from the raw URL
This is the fastest and safest method if you only want the content.
Returns unicode.
"""
try:
return self.readurl('http://%s/raw.php?i=%s' % (self.DOMAIN, _id), if_fail='raise').decode(self.ENCODING)
except BrowserHTTPNotFound:
raise PasteNotFound()
def post_paste(self, paste, expiration=None):
self.home()
if not self.is_on_page(PostPage):
self.home()
self.page.post(paste, expiration=expiration)
paste.id = self.page.get_id()
def api_post_paste(self, paste, expiration=None):
data = {'api_dev_key': self.api_key,
'api_option': 'paste',
'api_paste_code': paste.contents.encode(self.ENCODING),
}
if self.password:
data['api_user_key'] = self.api_login()
if paste.public is True:
data['api_paste_private'] = '0'
elif paste.public is False:
data['api_paste_private'] = '1'
if paste.title:
data['api_paste_name'] = paste.title.encode(self.ENCODING)
if expiration:
data['api_paste_expire_date'] = expiration
res = self.readurl(self.API_URL, urllib.urlencode(data)).decode(self.ENCODING)
self._validate_api_response(res)
paste.id = re.match('^%s$' % self.PASTE_URL, res).groupdict()['id']
def api_login(self):
# "The api_user_key does not expire."
# TODO store it on disk
if self.user_key:
return self.user_key
data = {'api_dev_key': self.api_key,
'api_user_name': self.username,
'api_user_password': self.password
}
res = self.readurl('http://%s/api/api_login.php' % self.DOMAIN,
urllib.urlencode(data)).decode(self.ENCODING)
try:
self._validate_api_response(res)
except BadAPIRequest, e:
if str(e) == 'invalid login':
raise BrowserIncorrectPassword
else:
raise e
self.user_key = res
return res
def _validate_api_response(self, res):
matches = re.match('Bad API request, (?P.+)', res)
if matches:
raise BadAPIRequest(matches.groupdict().get('error'))
def is_logged(self):
return self.page and self.page.is_logged()
def login(self):
self.location('http://%s/login' % self.DOMAIN, no_login=True)
self.page.login(self.username, self.password)
if not self.is_logged():
raise BrowserIncorrectPassword()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/pastebin/pages.py 0000664 0000000 0000000 00000006125 11666415431 0027161 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage, BrokenPageError
__all__ = ['PastePage', 'PostPage']
class BasePastebinPage(BasePage):
def is_logged(self):
header = self.parser.select(self.document.getroot(),
'id("header_bottom")/ul[@class="top_menu"]', 1, 'xpath')
for link in header.xpath('//ul/li/a'):
if link.text == 'logout':
return True
if link.text == 'login':
return False
# XXX hack, since all pages are detected as a PostPage we make PastePage
# inherit LoginPage
class LoginPage(BasePastebinPage):
def login(self, username, password):
self.browser.select_form(nr=1)
self.browser['user_name'] = username.encode(self.browser.ENCODING)
self.browser['user_password'] = password.encode(self.browser.ENCODING)
self.browser.submit()
class PastePage(LoginPage):
def fill_paste(self, paste):
header = self.parser.select(self.document.getroot(),
'id("content_left")//div[@class="paste_box_info"]', 1, 'xpath')
paste.title = self.parser.select(header,
'//div[@class="paste_box_line1"]//h1', 1, 'xpath').text
paste.contents = self.parser.select(self.document.getroot(),
'//textarea[@id="paste_code"]', 1, 'xpath').text
visibility_text = self.parser.select(header,
'//div[@class="paste_box_line1"]//img', 1, 'xpath').attrib['alt']
if visibility_text.startswith('Public'):
paste.public = True
elif visibility_text.startswith('Private'):
paste.public = False
else:
raise BrokenPageError('Unable to get the paste visibility')
return paste
def get_id(self):
"""
Find out the ID from the URL
"""
return self.group_dict['id']
class PostPage(BasePastebinPage):
def post(self, paste, expiration=None):
self.browser.select_form(name='myform')
self.browser['paste_code'] = paste.contents.encode(self.browser.ENCODING)
self.browser['paste_name'] = paste.title.encode(self.browser.ENCODING)
if paste.public is True:
self.browser['paste_private'] = ['0']
elif paste.public is False:
self.browser['paste_private'] = ['1']
if expiration:
self.browser['paste_expire_date'] = [expiration]
self.browser.submit()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/pastebin/paste.py 0000664 0000000 0000000 00000001650 11666415431 0027174 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.paste import BasePaste
__all__ = ['PastebinPaste']
class PastebinPaste(BasePaste):
@classmethod
def id2url(cls, _id):
return 'http://pastebin.com/%s' % _id
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/pastebin/test.py 0000664 0000000 0000000 00000007562 11666415431 0027047 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
from weboob.capabilities.base import NotLoaded
from weboob.capabilities.paste import PasteNotFound
class PastebinTest(BackendTest):
BACKEND = 'pastebin'
def test_get_paste(self):
for _id in ('7HmXwzyt', 'http://pastebin.com/7HmXwzyt'):
# html method
p = self.backend.get_paste(_id)
self.backend.fillobj(p, ['title'])
assert p.title == 'plop'
assert p.page_url == 'http://pastebin.com/7HmXwzyt'
assert p.contents == 'prout'
assert p.public is True
# raw method
p = self.backend.get_paste(_id)
self.backend.fillobj(p, ['contents'])
assert p.title is NotLoaded
assert p.page_url == 'http://pastebin.com/7HmXwzyt'
assert p.contents == 'prout'
assert p.public is NotLoaded
def test_post(self):
p = self.backend.new_paste(None, title='ouiboube', contents='Weboob Test', public=True)
self.backend.post_paste(p, max_age=600)
assert p.id
self.backend.fill_paste(p, ['title'])
assert p.title == 'ouiboube'
assert p.id in p.page_url
assert p.public is True
def test_specialchars(self):
# post a paste and get the contents through the HTML response
p1 = self.backend.new_paste(None, title='ouiboube', contents=u'Weboob ¿¡', public=False)
self.backend.post_paste(p1, max_age=600)
assert p1.id
assert p1.public is False
# this should use the raw method to get the contents
p2 = self.backend.get_paste(p1.id)
self.backend.fillobj(p2, ['contents'])
assert p2.contents == p1.contents
assert p2.public is NotLoaded
def test_notfound(self):
for _id in ('weboooooooooooooooooooooooooob', 'http://pastebin.com/weboooooooooooooooooooooooooob'):
# html method
p = self.backend.get_paste(_id)
self.assertRaises(PasteNotFound, self.backend.fillobj, p, ['title'])
# raw method
p = self.backend.get_paste(_id)
self.assertRaises(PasteNotFound, self.backend.fillobj, p, ['contents'])
def test_checkurl(self):
# call with an URL we can't handle with this backend
assert self.backend.get_paste('http://pastealacon.com/1') is None
def test_can_post(self):
assert self.backend.can_post('hello', public=None) > 0
assert self.backend.can_post('hello', public=True) > 0
assert self.backend.can_post('hello', public=False) > 0
assert self.backend.can_post('hello', public=True, max_age=600) > 0
assert self.backend.can_post('hello', public=True, max_age=3600*24) > 0
assert self.backend.can_post('hello', public=True, max_age=3600*24*3) > 0
assert self.backend.can_post('hello', public=True, max_age=False) > 0
assert self.backend.can_post('hello', public=None, max_age=False) > 0
assert self.backend.can_post('hello', public=True, max_age=3600*24*40) > 0
assert self.backend.can_post(u'héhé', public=True) > 0
assert self.backend.can_post(u'hello ♥', public=True) > 0
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/phpbb/ 0000775 0000000 0000000 00000000000 11666415431 0024772 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/phpbb/__init__.py 0000664 0000000 0000000 00000001502 11666415431 0027101 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .browser import PhpBB
from .backend import PhpBBBackend
__all__ = ['PhpBB', 'PhpBBBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/phpbb/backend.py 0000664 0000000 0000000 00000016262 11666415431 0026742 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.newsfeed import Newsfeed
from weboob.tools.value import Value, ValueInt, ValueBackendPassword
from weboob.tools.misc import limit
from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Message, Thread, CantSendMessage
from .browser import PhpBB
from .tools import rssid, url2id, id2url, id2topic
__all__ = ['PhpBBBackend']
class PhpBBBackend(BaseBackend, ICapMessages, ICapMessagesPost):
NAME = 'phpbb'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = "phpBB forum"
CONFIG = BackendConfig(Value('url', label='URL of forum', regexp='https?://.*'),
Value('username', label='Username', default=''),
ValueBackendPassword('password', label='Password', default=''),
ValueInt('thread_unread_messages', label='Limit number of unread messages to retrieve for a thread', default=500)
)
STORAGE = {'seen': {}}
BROWSER = PhpBB
def create_default_browser(self):
username = self.config['username'].get()
if len(username) > 0:
password = self.config['password'].get()
else:
password = None
return self.create_browser(self.config['url'].get(),
username, password)
#### ICapMessages ##############################################
def _iter_threads(self, root_link=None):
with self.browser:
links = list(self.browser.iter_links(root_link.url if root_link else None))
for link in links:
if link.type == link.FORUM:
link.title = '%s[%s]' % (root_link.title if root_link else '', link.title)
for thread in self._iter_threads(link):
yield thread
if link.type == link.TOPIC:
thread = Thread(url2id(link.url))
thread.title = ('%s ' % root_link.title if root_link else '') + link.title
thread.date = link.date
thread.nb_messages = link.nb_messages
thread.flags = thread.IS_DISCUSSION
yield thread
def iter_threads(self):
return self._iter_threads()
def get_thread(self, id):
thread = None
parent = None
if isinstance(id, Thread):
thread = id
id = thread.id
thread_id = url2id(id, nopost=True) or id
try:
last_seen_id = self.storage.get('seen', default={})[id2topic(thread_id)]
except KeyError:
last_seen_id = 0
with self.browser:
for post in self.browser.iter_posts(id):
if not thread:
thread = Thread(thread_id)
thread.title = post.title
m = self._post2message(thread, post)
m.parent = parent
if last_seen_id < post.id:
m.flags |= Message.IS_UNREAD
if parent:
parent.children = [m]
else:
thread.root = m
parent = m
return thread
def _post2message(self, thread, post):
signature = post.signature
if signature:
signature += ' '
signature += 'URL: %s' % self.browser.absurl(id2url('%s.%s' % (thread.id, post.id)))
return Message(thread=thread,
id=post.id,
title=post.title,
sender=post.author,
receivers=None,
date=post.date,
parent=None,
content=post.content,
signature=signature,
children=[],
flags=Message.IS_HTML)
def iter_unread_messages(self, thread=None):
with self.browser:
url = self.browser.get_root_feed_url()
for article in Newsfeed(url, rssid).iter_entries():
id = url2id(article.link)
thread = None
try:
last_seen_id = self.storage.get('seen', default={})[id2topic(id)]
except KeyError:
last_seen_id = 0
child = None
iterator = self.browser.riter_posts(id, last_seen_id)
if self.config['thread_unread_messages'].get() > 0:
iterator = limit(iterator, self.config['thread_unread_messages'].get())
for post in iterator:
if not thread:
thread = Thread('%s.%s' % (post.forum_id, post.topic_id))
message = self._post2message(thread, post)
if child:
message.children.append(child)
child.parent = message
if post.parent:
message.parent = Message(thread=thread,
id=post.parent)
else:
thread.root = message
yield message
def set_message_read(self, message):
try:
last_seen_id = self.storage.get('seen', default={})[id2topic(message.thread.id)]
except KeyError:
last_seen_id = 0
if message.id > last_seen_id:
self.storage.set('seen', id2topic(message.thread.id), message.id)
self.storage.save()
def fill_thread(self, thread, fields):
return self.get_thread(thread)
#### ICapMessagesReply #########################################
def post_message(self, message):
assert message.thread
forum = 0
topic = 0
if message.thread:
try:
if '.' in message.thread.id:
forum, topic = [int(i) for i in message.thread.id.split('.', 1)]
else:
forum = int(message.thread.id)
except ValueError:
raise CantSendMessage('Thread ID must be in form "FORUM_ID[.TOPIC_ID]".')
with self.browser:
return self.browser.post_answer(forum,
topic,
message.title,
message.content)
OBJECTS = {Thread: fill_thread}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/phpbb/browser.py 0000664 0000000 0000000 00000015032 11666415431 0027030 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
import urllib
from urlparse import urlsplit
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from weboob.capabilities.messages import CantSendMessage
from .pages.index import LoginPage
from .pages.forum import ForumPage, TopicPage, PostingPage
from .tools import id2url, url2id
__all__ = ['PhpBB']
# Browser
class PhpBB(BaseBrowser):
PAGES = {'https?://.*/index.php': ForumPage,
'https?://.*/': ForumPage,
'https?://.*/viewforum.php\?f=(\d+)': ForumPage,
'https?://.*/search.php\?.*': ForumPage,
'https?://.*/viewtopic.php\?.*': TopicPage,
'https?://.*/posting.php\?.*': PostingPage,
'https?://.*/ucp.php\?mode=login.*': LoginPage,
}
last_board_msg_id = None
def __init__(self, url, *args, **kwargs):
self.url = url
v = urlsplit(url)
self.PROTOCOL = v.scheme
self.DOMAIN = v.netloc
self.BASEPATH = v.path[:v.path.rfind('/')]
BaseBrowser.__init__(self, *args, **kwargs)
def absurl(self, rel):
return BaseBrowser.absurl(self, '%s/%s' % (self.BASEPATH, rel))
def home(self):
self.location(self.url)
def is_logged(self):
return not self.page or self.page.is_logged()
def login(self):
data = {'login': 'Connexion',
'username': self.username,
'password': self.password,
}
self.location('%s/ucp.php?mode=login' % self.BASEPATH, urllib.urlencode(data), no_login=True)
assert self.is_on_page(LoginPage)
if not self.page.is_logged():
raise BrowserIncorrectPassword(self.page.get_error_message())
def get_root_feed_url(self):
self.home()
return self.page.get_feed_url()
def iter_links(self, url):
if url:
self.location(url)
else:
self.home()
assert self.is_on_page(ForumPage)
return self.page.iter_links()
def iter_posts(self, id, stop_id=None):
if id.startswith('http'):
self.location(id)
else:
self.location('%s/%s' % (self.BASEPATH, id2url(id)))
assert self.is_on_page(TopicPage)
parent = 0
while 1:
for post in self.page.iter_posts():
if stop_id and post.id >= stop_id:
return
post.parent = parent
yield post
parent = post.id
if self.page.cur_page == self.page.tot_pages:
return
self.location(self.page.next_page_url())
def riter_posts(self, id, stop_id=None):
if id.startswith('http'):
self.location(id)
else:
self.location('%s/%s' % (self.BASEPATH, id2url(id)))
assert self.is_on_page(TopicPage)
child = None
while 1:
for post in self.page.riter_posts():
if child:
child.parent = post.id
yield child
if post.id <= stop_id:
return
child = post
if self.page.cur_page == 1:
if child:
yield child
return
self.location(self.page.prev_page_url())
def get_post(self, id):
if id.startswith('http'):
self.location(id)
id = url2id(id)
else:
self.location('%s/%s' % (self.BASEPATH, id2url(id)))
assert self.is_on_page(TopicPage)
post = self.page.get_post(int(id.split('.')[-1]))
if not post:
return None
if post.parent == 0 and self.page.cur_page > 1:
self.location(self.page.prev_page_url())
post.parent = self.page.get_last_post_id()
return post
def get_forums(self):
self.home()
return dict(self.page.iter_all_forums())
def post_answer(self, forum_id, topic_id, title, content):
if topic_id == 0:
if not forum_id:
forums = self.get_forums()
forums_prompt = 'Forums list:\n%s' % ('\n'.join(['\t- %s' % f for f in forums.itervalues()]))
m = re.match('\[(.*)\] (.*)', title or '')
if not m:
raise CantSendMessage('Please enter a title formatted like that:\n\t"[FORUM] SUBJECT"\n\n%s' % forums_prompt)
forum_id = None
for k,v in forums.iteritems():
if v.lower() == m.group(1).lower():
forum_id = k
break
if not forum_id:
raise CantSendMessage('Forum "%s" not found.\n\n%s' % (m.group(1), forums_prompt))
self.location('%s/posting.php?mode=post&f=%d' % (self.BASEPATH, forum_id))
assert self.is_on_page(PostingPage)
self.page.post(title, content)
assert self.is_on_page(PostingPage)
error = self.page.get_error_message()
if error:
raise CantSendMessage(u'Unable to send message: %s' % error)
else:
self.location('%s/%s' % (self.BASEPATH, id2url(topic_id)))
assert self.is_on_page(TopicPage)
self.page.go_reply()
assert self.is_on_page(PostingPage)
# Don't send title because it isn't needed in real use case
# and with monboob title is something like:
# Re: [Forum Name] Re: Topic Name
if title is not None and title.startswith('Re:'):
title = None
self.page.post(title, content)
assert self.is_on_page(PostingPage)
error = self.page.get_error_message()
if error:
raise CantSendMessage(u'Unable to send message: %s' % error)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/phpbb/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026071 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/phpbb/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0030170 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/phpbb/pages/forum.py 0000664 0000000 0000000 00000020217 11666415431 0027575 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from time import sleep
from urlparse import urlsplit, parse_qs
from weboob.tools.browser import BrokenPageError
from .index import PhpBBPage
from ..tools import parse_date
__all__ = ['Link', 'ForumPage', 'TopicPage', 'PostingPage']
class Link(object):
(FORUM,
TOPIC) = xrange(2)
def __init__(self, type, url):
self.type = type
self.url = url
self.title = u''
self.date = None
self.nb_messages = 0
class ForumPage(PhpBBPage):
def iter_links(self):
for li in self.parser.select(self.document.getroot(), 'ul.forums li.row'):
title = li.cssselect('a.forumtitle')[0]
link = Link(Link.FORUM, title.attrib['href'])
link.title = title.text.strip()
yield link
for li in self.parser.select(self.document.getroot(), 'ul.topics li.row'):
title = li.cssselect('a.topictitle')[0]
link = Link(Link.TOPIC, title.attrib['href'])
link.title = title.text.strip()
for a in li.find('dl').find('dt').findall('a'):
for text in (a.text, a.tail):
if text is None:
continue
try:
link.date = parse_date(text.strip(u'» \r\n'))
except ValueError:
continue
else:
break
# it only lists number of answers, so we add 1.
link.nb_messages = int(li.cssselect('dd.posts')[0].text.strip()) + 1
yield link
def iter_all_forums(self):
for option in self.parser.select(self.document.getroot(), 'select#f', 1).findall('option'):
value = int(option.attrib['value'])
if value < 0 or not option.text:
continue
yield value, option.text.strip(u'» \xa0\n\r')
class Post(object):
def __init__(self, forum_id, topic_id, id):
self.id = int(id)
self.forum_id = forum_id
self.topic_id = topic_id
self.title = u''
self.author = u''
self.date = None
self.content = u''
self.signature = u''
self.parent = 0
class TopicPage(PhpBBPage):
def on_loaded(self):
div = self.document.getroot().cssselect('div.pagination')[0]
strongs = div.cssselect('strong')
self.cur_page = int(strongs[0].text.strip())
self.tot_pages = int(strongs[1].text.strip())
try:
url = self.document.xpath('//h2/a')[-1].attrib['href']
except BrokenPageError:
url = self.url
v = urlsplit(url)
args = parse_qs(v.query)
self.topic_id = int(args['t'][0])
self.forum_id = int(args['f'][0]) if 'f' in args else 0
self.forum_title = u''
nav = self.parser.select(self.document.getroot(), 'li.icon-home')
if len(nav) > 0:
text = nav[0].findall('a')[-1].text.strip()
if len(text) >= 20:
text = text[:20] + u'…'
self.forum_title = '[%s] ' % text
def go_reply(self):
self.browser.follow_link(url_regex='posting\.php')
def next_page_url(self):
try:
return self.parser.select(self.document.getroot(), 'a.right-box', 1).attrib['href']
except BrokenPageError:
a_list = self.parser.select(self.document.getroot(), 'div.pagination', 1).findall('a')
if self.cur_page == self.tot_pages:
return '#'
return a_list[-1].attrib['href']
def prev_page_url(self):
try:
return self.parser.select(self.document.getroot(), 'a.left-box', 1).attrib['href']
except BrokenPageError:
a_list = self.parser.select(self.document.getroot(), 'div.pagination', 1).findall('a')
if self.cur_page == self.tot_pages:
a = a_list[-1]
else:
a = a_list[-2]
return a.attrib['href']
def iter_posts(self):
for div in self.parser.select(self.document.getroot(), 'div.post'):
yield self._get_post(div)
def riter_posts(self):
for div in reversed(self.parser.select(self.document.getroot(), 'div.post')):
yield self._get_post(div)
def get_post(self, id):
parent = 0
for div in self.parser.select(self.document.getroot(), 'div.post'):
if div.attrib['id'] == 'p%d' % id:
post = self._get_post(div)
post.parent = parent
return post
else:
parent = int(div.attrib['id'][1:])
def _get_post(self, div):
body = div.cssselect('div.postbody')[0]
profile = div.cssselect('dl.postprofile')[0]
id = div.attrib['id'][1:]
post = Post(self.forum_id, self.topic_id, id)
title_tags = body.xpath('//h3/a')
if len(title_tags) == 0:
title_tags = self.document.xpath('//h2/a')
if len(title_tags) == 0:
title = u''
self.logger.warning('Unable to parse title')
else:
title = title_tags[-1].text.strip()
post.title = self.forum_title + title
for a in profile.cssselect('dt a'):
if a.text:
post.author = a.text.strip()
p_tags = body.cssselect('p.author')
if len(p_tags) == 0:
p_tags = body.find('p')
if len(p_tags) == 0:
post.date = None
self.logger.warning('Unable to parse datetime')
else:
p = p_tags[0]
text = p.find('strong') is not None and p.find('strong').tail
if not text:
text = p.text[4:]
text = text.strip(u'» \n\r')
try:
post.date = parse_date(text)
except ValueError:
self.logger.warning(u'Unable to parse datetime "%s"' % text)
post.content = self.parser.tostring(body.cssselect('div.content')[0])
signature = body.cssselect('div.signature')
if len(signature) > 0:
post.signature = self.parser.tostring(signature[0])
return post
def get_last_post_id(self):
id = 0
for div in self.parser.select(self.document.getroot(), 'div.post'):
id = int(div.attrib['id'][1:])
return id
class PostingPage(PhpBBPage):
def post(self, title, content):
self.browser.select_form(predicate=lambda form: form.attrs.get('id', '') == 'postform')
self.browser.set_all_readonly(False)
if title:
self.browser['subject'] = title.encode('utf-8')
self.browser['message'] = content.encode('utf-8')
# This code on phpbb:
# if ($cancel || ($current_time - $lastclick < 2 && $submit))
# {
# /* ... */
# redirect($redirect);
# }
# To prevent that shit because weboob is too fast, we simulate
# a value of lastclick 10 seconds before.
self.browser['lastclick'] = str(int(self.browser['lastclick']) - 10)
# Likewise for create_time, with this check:
# $diff = time() - $creation_time;
# // If creation_time and the time() now is zero we can assume it was not a human doing this (the check for if ($diff)...
# if ($diff && ($diff <= $timespan || $timespan === -1))
# But as the form_token depends on the create_time value, I can't
# change it. But I can wait a second before posting...
sleep(1)
self.browser.submit(name='post')
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/phpbb/pages/index.py 0000664 0000000 0000000 00000002506 11666415431 0027555 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage
class PhpBBPage(BasePage):
def is_logged(self):
return len(self.document.getroot().cssselect('li.icon-register')) == 0
def get_feed_url(self):
links = self.document.getroot().cssselect('link[type="application/atom+xml"]')
return links[-1].attrib['href']
def get_error_message(self):
errors = []
for div in self.parser.select(self.document.getroot(), 'div.error,p.error'):
if div.text:
errors.append(div.text.strip())
return ', '.join(errors)
class LoginPage(PhpBBPage):
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/phpbb/test.py 0000664 0000000 0000000 00000002055 11666415431 0026325 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
__all__ = ['PhpBBTest']
class PhpBBTest(BackendTest):
BACKEND = 'phpbb'
def testthreads(self):
for thread in self.backend.iter_threads():
pass
def test_unread_messages(self):
for message in self.backend.iter_unread_messages():
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/phpbb/tools.py 0000664 0000000 0000000 00000004375 11666415431 0026515 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from dateutil.parser import parse as _parse_dt
from urlparse import urlsplit, parse_qs
from weboob.tools.misc import local2utc
def url2id(url, nopost=False):
v = urlsplit(url)
pagename = v.path.split('/')[-1]
args = parse_qs(v.query)
if pagename == 'viewforum.php':
return '%d' % int(args['f'][0])
if pagename == 'viewtopic.php':
if 'f' in args:
s = '%d' % int(args['f'][0])
else:
s = '0'
s += '.%d' % int(args['t'][0])
if 'p' in args and not nopost:
s += '.%d' % int(args['p'][0])
return s
return None
def id2url(id):
v = id.split('.')
if len(v) == 1:
return 'viewforum.php?f=%d' % int(v[0])
if len(v) == 2:
return 'viewtopic.php?f=%d&t=%d' % (int(v[0]), int(v[1]))
if len(v) == 3:
return 'viewtopic.php?f=%d&t=%d&p=%d#p%d' % (int(v[0]),
int(v[1]),
int(v[2]),
int(v[2]))
def id2topic(id):
try:
return int(id.split('.')[1])
except IndexError:
return None
def rssid(id):
return id
def parse_date(s):
s = s.replace(u'Fév', 'Feb') \
.replace(u'Avr', 'Apr') \
.replace(u'Mai', 'May') \
.replace(u'Juin', 'Jun') \
.replace(u'Juil', 'Jul') \
.replace(u'Aoû', 'Aug') \
.replace(u'Ao\xfbt', 'Aug') \
.replace(u'Déc', 'Dec')
return local2utc(_parse_dt(s))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/piratebay/ 0000775 0000000 0000000 00000000000 11666415431 0025657 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/piratebay/__init__.py 0000664 0000000 0000000 00000000106 11666415431 0027765 0 ustar 00root root 0000000 0000000 from .backend import PiratebayBackend
__all__ = ['PiratebayBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/piratebay/backend.py 0000664 0000000 0000000 00000003161 11666415431 0027621 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.torrent import ICapTorrent
from weboob.tools.backend import BaseBackend
from .browser import PiratebayBrowser
__all__ = ['PiratebayBackend']
class PiratebayBackend(BaseBackend, ICapTorrent):
NAME = 'piratebay'
MAINTAINER = 'Julien Veyssier'
EMAIL = 'julien.veyssier@aiur.fr'
VERSION = '0.9.1'
DESCRIPTION = 'the pirate bay bittorrent tracker'
LICENSE = 'AGPLv3+'
BROWSER = PiratebayBrowser
def create_default_browser(self):
return self.create_browser()
def get_torrent(self, id):
return self.browser.get_torrent(id)
def get_torrent_file(self, id):
torrent = self.browser.get_torrent(id)
if not torrent:
return None
return self.browser.openurl(torrent.url.encode('utf-8')).read()
def iter_torrents(self, pattern):
return self.browser.iter_torrents(pattern.replace(' ','+'))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/piratebay/browser.py 0000664 0000000 0000000 00000003400 11666415431 0027711 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import urllib
from weboob.tools.browser import BaseBrowser
from .pages.index import IndexPage
from .pages.torrents import TorrentsPage, TorrentPage
__all__ = ['PiratebayBrowser']
class PiratebayBrowser(BaseBrowser):
DOMAIN = 'thepiratebay.org'
PROTOCOL = 'https'
ENCODING = 'utf-8'
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
PAGES = {'https://thepiratebay.org' : IndexPage,
'https://thepiratebay.org/search/.*/0/7/0' : TorrentsPage,
'https://thepiratebay.org/torrent/.*' : TorrentPage
}
def home(self):
return self.location('https://thepiratebay.org')
def iter_torrents(self, pattern):
self.location('https://thepiratebay.org/search/%s/0/7/0' % urllib.quote_plus(pattern.encode('utf-8')))
assert self.is_on_page(TorrentsPage)
return self.page.iter_torrents()
def get_torrent(self, id):
self.location('https://thepiratebay.org/torrent/%s/' % id)
assert self.is_on_page(TorrentPage)
return self.page.get_torrent(id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/piratebay/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026756 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/piratebay/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0031055 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/piratebay/pages/index.py 0000664 0000000 0000000 00000001626 11666415431 0030444 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage
__all__ = ['IndexPage']
class IndexPage(BasePage):
def is_logged(self):
return 'id' in self.document.find('body').attrib
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/piratebay/pages/torrents.py 0000664 0000000 0000000 00000007366 11666415431 0031224 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage
from weboob.capabilities.torrent import Torrent
__all__ = ['TorrentsPage']
class TorrentsPage(BasePage):
def unit(self, n, u):
m = {'B': 1,
'KB': 1024,
'MB': 1024*1024,
'GB': 1024*1024*1024,
'TB': 1024*1024*1024*1024,
}
#return float(n.replace(',', '')) * m.get(u, 1)
return float(n*m[u])
def iter_torrents(self):
table = self.parser.select(self.document.getroot(), 'table#searchResult', 1)
for tr in table.getiterator('tr'):
if tr.get('class','') != "header":
td = tr.getchildren()[1]
div = td.getchildren()[0]
link = div.find('a').attrib['href']
title = div.find('a').text
idt = link.split('/')[2]
a = td.getchildren()[1]
url = a.attrib['href']
size = td.find('font').text.split(',')[1].strip()
u = size.split(' ')[1].split(u'\xa0')[1].replace('i','')
size = size.split(' ')[1].split(u'\xa0')[0]
seed = tr.getchildren()[2].text
leech = tr.getchildren()[3].text
torrent = Torrent(idt,
title,
url=url,
size=self.unit(float(size),u),
seeders=int(seed),
leechers=int(leech))
yield torrent
class TorrentPage(BasePage):
def get_torrent(self, id):
for div in self.document.getiterator('div'):
if div.attrib.get('id','') == 'title':
title = div.text.strip()
elif div.attrib.get('class','') == 'download':
url = div.getchildren()[0].attrib.get('href','')
elif div.attrib.get('id','') == 'details':
size = float(div.getchildren()[0].getchildren()[5].text.split('(')[1].split('Bytes')[0])
if len(div.getchildren()) > 1 \
and div.getchildren()[1].attrib.get('class','') == 'col2' :
child_to_explore = div.getchildren()[1]
else:
child_to_explore = div.getchildren()[0]
prev_child_txt = "none"
seed="-1"
leech="-1"
for ch in child_to_explore.getchildren():
if prev_child_txt == "Seeders:":
seed = ch.text
if prev_child_txt == "Leechers:":
leech = ch.text
prev_child_txt = ch.text
elif div.attrib.get('class','') == 'nfo':
description = div.getchildren()[0].text
torrent = Torrent(id, title)
torrent.url = url
torrent.size = size
torrent.seeders = int(seed)
torrent.leechers = int(leech)
torrent.description = description
torrent.files = ['NYI']
return torrent
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/piratebay/test.py 0000664 0000000 0000000 00000001746 11666415431 0027220 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class PiratebayTest(BackendTest):
BACKEND = 'piratebay'
def test_torrent(self):
l = list(self.backend.iter_torrents('debian'))
if len(l) > 0:
self.backend.get_torrent_file(l[0].id)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/redmine/ 0000775 0000000 0000000 00000000000 11666415431 0025322 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/redmine/__init__.py 0000664 0000000 0000000 00000001442 11666415431 0027434 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import RedmineBackend
__all__ = ['RedmineBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/redmine/backend.py 0000664 0000000 0000000 00000023325 11666415431 0027270 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.content import ICapContent, Content
from weboob.capabilities.bugtracker import ICapBugTracker, Issue, Project, User, \
Version, Status, Update, Attachment, \
Query, Change
from weboob.capabilities.collection import ICapCollection, Collection, CollectionNotFound
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.browser import BrowserHTTPNotFound
from weboob.tools.value import ValueBackendPassword, Value
from .browser import RedmineBrowser
__all__ = ['RedmineBackend']
class RedmineBackend(BaseBackend, ICapContent, ICapBugTracker, ICapCollection):
NAME = 'redmine'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
DESCRIPTION = 'The Redmine project management web application'
LICENSE = 'AGPLv3+'
CONFIG = BackendConfig(Value('url', label='URL of the Redmine website'),
Value('username', label='Login'),
ValueBackendPassword('password', label='Password'))
BROWSER = RedmineBrowser
def create_default_browser(self):
return self.create_browser(self.config['url'].get(),
self.config['username'].get(),
self.config['password'].get())
############# CapContent ######################################################
def id2path(self, id):
return id.split('/', 2)
def get_content(self, id):
if isinstance(id, basestring):
content = Content(id)
else:
content = id
id = content.id
try:
_type, project, page = self.id2path(id)
except ValueError:
return None
with self.browser:
data = self.browser.get_wiki_source(project, page)
content.content = data
return content
def push_content(self, content, message=None, minor=False):
try:
_type, project, page = self.id2path(content.id)
except ValueError:
return
with self.browser:
return self.browser.set_wiki_source(project, page, content.content, message)
def get_content_preview(self, content):
try:
_type, project, page = self.id2path(content.id)
except ValueError:
return
with self.browser:
return self.browser.get_wiki_preview(project, page, content.content)
############# CapCollection ###################################################
def iter_resources(self, path):
if len(path) == 0:
return [Collection(project.id) for project in self.iter_projects()]
if len(path) == 1:
query = Query()
query.project = unicode(path[0])
return self.iter_issues(query)
raise CollectionNotFound()
############# CapBugTracker ###################################################
def _build_project(self, project_dict):
project = Project(project_dict['name'], project_dict['name'])
project.members = [User(int(u[0]), u[1]) for u in project_dict['members']]
project.versions = [Version(int(v[0]), v[1]) for v in project_dict['versions']]
project.categories = [c[1] for c in project_dict['categories']]
# TODO set the value of status
project.statuses = [Status(int(s[0]), s[1], 0) for s in project_dict['statuses']]
return project
def iter_issues(self, query):
"""
Iter issues with optionnal patterns.
@param query [Query]
@return [iter(Issue)] issues
"""
# TODO link between text and IDs.
kwargs = {'subject': query.title,
'author_id': query.author,
'assigned_to_id': query.assignee,
'fixed_version_id': query.version,
'category_id': query.category,
'status_id': query.status,
}
r = self.browser.query_issues(query.project, **kwargs)
project = self._build_project(r['project'])
for issue in r['iter']:
obj = Issue(issue['id'])
obj.project = project
obj.title = issue['subject']
obj.creation = issue['created_on']
obj.updated = issue['updated_on']
if isinstance(issue['author'], tuple):
obj.author = project.find_user(*issue['author'])
else:
obj.author = User(0, issue['author'])
if isinstance(issue['assigned_to'], tuple):
obj.assignee = project.find_user(*issue['assigned_to'])
else:
obj.assignee = issue['assigned_to']
obj.category = issue['category']
if issue['fixed_version'] is not None:
obj.version = project.find_version(*issue['fixed_version'])
else:
obj.version = None
obj.status = project.find_status(issue['status'])
yield obj
def get_issue(self, issue):
if isinstance(issue, Issue):
id = issue.id
else:
id = issue
issue = Issue(issue)
try:
with self.browser:
params = self.browser.get_issue(id)
except BrowserHTTPNotFound:
return None
issue.project = self._build_project(params['project'])
issue.title = params['subject']
issue.body = params['body']
issue.creation = params['created_on']
issue.updated = params['updated_on']
issue.attachments = []
for a in params['attachments']:
attachment = Attachment(a['id'])
attachment.filename = a['filename']
attachment.url = a['url']
issue.attachments.append(attachment)
issue.history = []
for u in params['updates']:
update = Update(u['id'])
update.author = issue.project.find_user(*u['author'])
update.date = u['date']
update.message = u['message']
update.changes = []
for i, (field, last, new) in enumerate(u['changes']):
change = Change(i)
change.field = field
change.last = last
change.new = new
update.changes.append(change)
issue.history.append(update)
issue.author = issue.project.find_user(*params['author'])
issue.assignee = issue.project.find_user(*params['assignee'])
issue.category = params['category'][1]
issue.version = issue.project.find_version(*params['version'])
issue.status = issue.project.find_status(params['status'][1])
return issue
def create_issue(self, project):
try:
with self.browser:
r = self.browser.query_issues(project)
except BrowserHTTPNotFound:
return None
issue = Issue(0)
issue.project = self._build_project(r['project'])
return issue
def post_issue(self, issue):
project = issue.project.id
kwargs = {'title': issue.title,
'version': issue.version.id if issue.version else None,
'assignee': issue.assignee.id if issue.assignee else None,
'category': issue.category,
'status': issue.status.id if issue.status else None,
'body': issue.body,
}
with self.browser:
if int(issue.id) < 1:
id = self.browser.create_issue(project, **kwargs)
else:
id = self.browser.edit_issue(issue.id, **kwargs)
if id is None:
return None
issue.id = id
return issue
def update_issue(self, issue, update):
if isinstance(issue, Issue):
issue = issue.id
with self.browser:
if update.hours:
return self.browser.logtime_issue(issue, update.hours, update.message)
else:
return self.browser.comment_issue(issue, update.message)
def remove_issue(self, issue):
"""
Remove an issue.
"""
if isinstance(issue, Issue):
issue = issue.id
with self.browser:
return self.browser.remove_issue(issue)
def iter_projects(self):
"""
Iter projects.
@return [iter(Project)] projects
"""
with self.browser:
for project in self.browser.iter_projects():
yield Project(project['id'], project['name'])
def get_project(self, id):
try:
with self.browser:
params = self.browser.get_issue(id)
except BrowserHTTPNotFound:
return None
return self._build_project(params['project'])
def fill_issue(self, issue, fields):
# currently there isn't cases where an Issue is uncompleted.
return issue
OBJECTS = {Issue: fill_issue}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/redmine/browser.py 0000664 0000000 0000000 00000017402 11666415431 0027363 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from urlparse import urlsplit
import urllib
import lxml.html
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from .pages.index import LoginPage, IndexPage, MyPage, ProjectsPage
from .pages.wiki import WikiPage, WikiEditPage
from .pages.issues import IssuesPage, IssuePage, NewIssuePage, IssueLogTimePage, \
IssueTimeEntriesPage
__all__ = ['RedmineBrowser']
# Browser
class RedmineBrowser(BaseBrowser):
ENCODING = 'utf-8'
PAGES = {'https?://[^/]+/': IndexPage,
'https?://[^/]+/login': LoginPage,
# compatibility with redmine 0.9
'https?://[^/]+/login\?back_url.*': MyPage,
'https?://[^/]+/my/page': MyPage,
'https?://[^/]+/projects': ProjectsPage,
'https?://[^/]+/projects/([\w-]+)/wiki/([^\/]+)/edit': WikiEditPage,
'https?://[^/]+/projects/[\w-]+/wiki/[^\/]*': WikiPage,
'https?://[^/]+/projects/[\w-]+/issues/new': NewIssuePage,
'https?://[^/]+/projects/[\w-]+/issues': IssuesPage,
'https?://[^/]+/issues(|/?\?.*)': IssuesPage,
'https?://[^/]+/issues/(\d+)': IssuePage,
'https?://[^/]+/issues/(\d+)/time_entries/new': IssueLogTimePage,
'https?://[^/]+/projects/[\w-]+/time_entries': IssueTimeEntriesPage,
}
def __init__(self, url, *args, **kwargs):
self._userid = 0
v = urlsplit(url)
self.PROTOCOL = v.scheme
self.DOMAIN = v.netloc
self.BASEPATH = v.path
if self.BASEPATH.endswith('/'):
self.BASEPATH = self.BASEPATH[:-1]
BaseBrowser.__init__(self, *args, **kwargs)
self.projects = {}
def is_logged(self):
return self.is_on_page(LoginPage) or self.page and len(self.page.document.getroot().cssselect('a.my-account')) == 1
def login(self):
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
if not self.is_on_page(LoginPage):
self.location('%s/login' % self.BASEPATH, no_login=True)
self.page.login(self.username, self.password)
if self.is_on_page(LoginPage):
raise BrowserIncorrectPassword()
divs = self.page.document.getroot().cssselect('div#loggedas')
if len(divs) > 0:
parts = divs[0].find('a').attrib['href'].split('/')
self._userid = int(parts[2])
def get_userid(self):
return self._userid
def get_wiki_source(self, project, page):
self.location('%s/projects/%s/wiki/%s/edit' % (self.BASEPATH, project, urllib.quote(page.encode('utf-8'))))
return self.page.get_source()
def set_wiki_source(self, project, page, data, message):
self.location('%s/projects/%s/wiki/%s/edit' % (self.BASEPATH, project, urllib.quote(page.encode('utf-8'))))
self.page.set_source(data, message)
def get_wiki_preview(self, project, page, data):
if (not self.is_on_page(WikiEditPage) or self.page.groups[0] != project
or self.page.groups[1] != page):
self.location('%s/projects/%s/wiki/%s/edit' % (self.BASEPATH,
project, urllib.quote(page.encode('utf-8'))))
url = '%s/projects/%s/wiki/%s/preview' % (self.BASEPATH, project, urllib.quote(page.encode('utf-8')))
params = {}
params['content[text]'] = data.encode('utf-8')
params['authenticity_token'] = "%s" % self.page.get_authenticity_token()
preview_html = lxml.html.fragment_fromstring(self.readurl(url,
urllib.urlencode(params)),
create_parent='div')
preview_html.find("fieldset").drop_tag()
preview_html.find("legend").drop_tree()
return lxml.html.tostring(preview_html)
def query_issues(self, project_name, **kwargs):
self.location('/projects/%s/issues' % project_name)
token = self.page.get_authenticity_token()
data = (('project_id', project_name),
('query[column_names][]', 'tracker'),
('authenticity_token', token),
('query[column_names][]', 'status'),
('query[column_names][]', 'priority'),
('query[column_names][]', 'subject'),
('query[column_names][]', 'assigned_to'),
('query[column_names][]', 'updated_on'),
('query[column_names][]', 'category'),
('query[column_names][]', 'fixed_version'),
('query[column_names][]', 'done_ratio'),
('query[column_names][]', 'author'),
('query[column_names][]', 'start_date'),
('query[column_names][]', 'due_date'),
('query[column_names][]', 'estimated_hours'),
('query[column_names][]', 'created_on'),
)
for key, value in kwargs.iteritems():
if value:
data += (('values[%s][]' % key, value),)
data += (('fields[]', key),)
data += (('operators[%s]' % key, '~'),)
self.location('/issues?set_filter=1&per_page=100', urllib.urlencode(data))
assert self.is_on_page(IssuesPage)
return {'project': self.page.get_project(project_name),
'iter': self.page.iter_issues(),
}
def get_issue(self, id):
self.location('/issues/%s' % id)
assert self.is_on_page(IssuePage)
return self.page.get_params()
def logtime_issue(self, id, hours, message):
self.location('/issues/%s/time_entries/new' % id)
assert self.is_on_page(IssueLogTimePage)
self.page.logtime(hours.seconds/3600, message)
def comment_issue(self, id, message):
self.location('/issues/%s' % id)
assert self.is_on_page(IssuePage)
self.page.fill_form(note=message)
def create_issue(self, project, **kwargs):
self.location('/projects/%s/issues/new' % project)
assert self.is_on_page(NewIssuePage)
self.page.fill_form(**kwargs)
assert self.is_on_page(IssuePage)
return int(self.page.groups[0])
def edit_issue(self, id, **kwargs):
self.location('/issues/%s' % id)
assert self.is_on_page(IssuePage)
self.page.fill_form(**kwargs)
assert self.is_on_page(IssuePage)
return int(self.page.groups[0])
def remove_issue(self, id):
self.location('/issues/%s' % id)
assert self.is_on_page(IssuePage)
token = self.page.get_authenticity_token()
data = (('authenticity_token', token),)
self.openurl('/issues/%s/destroy' % id, urllib.urlencode(data))
def iter_projects(self):
self.location('/projects')
return self.page.iter_projects()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/redmine/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026421 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/redmine/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0030520 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/redmine/pages/index.py 0000664 0000000 0000000 00000002657 11666415431 0030114 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage
class LoginPage(BasePage):
def login(self, username, password):
self.browser.select_form(nr=1)
self.browser['username'] = username
self.browser['password'] = password
self.browser.submit()
class IndexPage(BasePage):
pass
class MyPage(BasePage):
pass
class ProjectsPage(BasePage):
def iter_projects(self):
for ul in self.parser.select(self.document.getroot(), 'ul.projects'):
for li in ul.findall('li'):
prj = {}
link = li.find('div').find('a')
prj['id'] = link.attrib['href'].split('/')[-1]
prj['name'] = link.text
yield prj
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/redmine/pages/issues.py 0000664 0000000 0000000 00000030134 11666415431 0030307 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
import datetime
from weboob.capabilities.bugtracker import IssueError
from weboob.tools.browser import BasePage, BrokenPageError
from weboob.tools.misc import to_unicode
from weboob.tools.mech import ClientForm
class BaseIssuePage(BasePage):
def parse_datetime(self, text):
m = re.match('(\d+)/(\d+)/(\d+) (\d+):(\d+) (\w+)', text)
if m:
date = datetime.datetime(int(m.group(3)),
int(m.group(1)),
int(m.group(2)),
int(m.group(4)),
int(m.group(5)))
if m.group(6) == 'pm':
date += datetime.timedelta(0,12*3600)
return date
m = re.match('(\d+)-(\d+)-(\d+) (\d+):(\d+)', text)
if m:
return datetime.datetime(int(m.group(1)),
int(m.group(2)),
int(m.group(3)),
int(m.group(4)),
int(m.group(5)))
self.logger.warning('Unable to parse "%s"' % text)
return text
PROJECT_FIELDS = {'members': 'values_assigned_to_id',
'categories': 'values_category_id',
'versions': 'values_fixed_version_id',
'statuses': 'values_status_id',
}
def iter_choices(self, name):
try:
select = self.parser.select(self.document.getroot(), 'select#%s' % name, 1)
except BrokenPageError:
return
for option in select.findall('option'):
if option.attrib['value'].isdigit():
yield (option.attrib['value'], option.text)
def get_project(self, project_name):
project = {}
project['name'] = project_name
for field, elid in self.PROJECT_FIELDS.iteritems():
project[field] = list(self.iter_choices(elid))
return project
def get_authenticity_token(self):
tokens = self.parser.select(self.document.getroot(), 'input[name=authenticity_token]')
if len(tokens) == 0:
raise IssueError("You doesn't have rights to remove this issue.")
token = tokens[0].attrib['value']
return token
class IssuesPage(BaseIssuePage):
PROJECT_FIELDS = {'members': 'values_assigned_to_id',
'categories': 'values_category_id',
'versions': 'values_fixed_version_id',
'statuses': 'values_status_id',
}
def iter_issues(self):
try:
issues = self.parser.select(self.document.getroot(), 'table.issues', 1)
except BrokenPageError:
# No results.
return
for tr in issues.getiterator('tr'):
if not tr.attrib.get('id', '').startswith('issue-'):
continue
issue = {}
for td in tr.getiterator('td'):
field = td.attrib.get('class', '')
if field in ('checkbox','todo',''):
continue
a = td.find('a')
if a is not None:
if a.attrib['href'].startswith('/users/') or \
a.attrib['href'].startswith('/versions/'):
text = (int(a.attrib['href'].split('/')[-1]), a.text)
else:
text = a.text
else:
text = td.text
if field.endswith('_on'):
text = self.parse_datetime(text)
elif field.endswith('_date') and text is not None:
m = re.match('(\d+)-(\d+)-(\d+)', text)
if m:
text = datetime.datetime(int(m.group(1)),
int(m.group(2)),
int(m.group(3)))
if isinstance(text, str):
text = to_unicode(text)
issue[field] = text
if len(issue) != 0:
yield issue
class NewIssuePage(BaseIssuePage):
PROJECT_FIELDS = {'members': 'issue_assigned_to_id',
'categories': 'issue_category_id',
'versions': 'issue_fixed_version_id',
'statuses': 'issue_status_id',
}
def set_title(self, title):
self.browser['issue[subject]'] = title.encode('utf-8')
def set_body(self, body):
self.browser['issue[description]'] = body.encode('utf-8')
def set_assignee(self, member):
if member:
self.browser['issue[assigned_to_id]'] = [str(member)]
else:
self.browser['issue[assigned_to_id]'] = ['']
def set_version(self, version):
try:
if version:
self.browser['issue[fixed_version_id]'] = [str(version)]
else:
self.browser['issue[fixed_version_id]'] = ['']
except ClientForm.ItemNotFoundError:
self.logger.warning('Version not found: %s' % version)
def set_category(self, category):
if category:
select = self.parser.select(self.document.getroot(), 'select#issue_category_id', 1)
for option in select.findall('option'):
if option.text and option.text.strip() == category:
self.browser['issue[category_id]'] = [option.attrib['value']]
return
self.logger.warning('Category "%s" not found' % category)
else:
self.browser['issue[category_id]'] = ['']
def set_status(self, status):
assert status is not None
self.browser['issue[status_id]'] = [str(status)]
def set_note(self, message):
self.browser['notes'] = message.encode('utf-8')
def fill_form(self, **kwargs):
self.browser.select_form(predicate=lambda form: form.attrs.get('id', '') == 'issue-form')
for key, value in kwargs.iteritems():
if value is not None:
getattr(self, 'set_%s' % key)(value)
self.browser.submit()
class IssuePage(NewIssuePage):
def _parse_selection(self, id):
try:
select = self.parser.select(self.document.getroot(), 'select#%s' % id, 1)
except BrokenPageError:
# not available for this project
return ('', None)
else:
options = select.findall('option')
for option in options:
if 'selected' in option.attrib:
return (int(option.attrib['value']), to_unicode(option.text))
return ('', None)
def get_params(self):
params = {}
content = self.parser.select(self.document.getroot(), 'div#content', 1)
issue = self.parser.select(content, 'div.issue', 1)
params['project'] = self.get_project(to_unicode(self.parser.select(self.document.getroot(), 'h1', 1).text))
params['subject'] = to_unicode(self.parser.select(issue, 'div.subject', 1).find('div').find('h3').text.strip())
params['body'] = to_unicode(self.parser.select(self.document.getroot(), 'textarea#issue_description', 1).text)
author = self.parser.select(issue, 'p.author', 1)
# check issue 666 on symlink.me
i = 0
alist = author.findall('a')
if not 'title' in alist[i].attrib:
params['author'] = (int(alist[i].attrib['href'].split('/')[-1]),
to_unicode(alist[i].text))
i += 1
else:
params['author'] = (0, 'Anonymous')
params['created_on'] = self.parse_datetime(alist[i].attrib['title'])
if len(alist) > i+1:
params['updated_on'] = self.parse_datetime(alist[i+1].attrib['title'])
else:
params['updated_on'] = None
params['status'] = self._parse_selection('issue_status_id')
params['assignee'] = self._parse_selection('issue_assigned_to_id')
params['category'] = self._parse_selection('issue_category_id')
params['version'] = self._parse_selection('issue_fixed_version_id')
params['attachments'] = []
try:
for p in self.parser.select(content, 'div.attachments', 1).findall('p'):
attachment = {}
a = p.find('a')
attachment['id'] = int(a.attrib['href'].split('/')[-2])
attachment['filename'] = p.find('a').text
attachment['url'] = '%s://%s%s' % (self.browser.PROTOCOL, self.browser.DOMAIN, p.find('a').attrib['href'])
params['attachments'].append(attachment)
except BrokenPageError:
pass
params['updates'] = []
for div in self.parser.select(content, 'div.journal'):
update = {}
update['id'] = div.find('h4').find('div').find('a').text[1:]
alist = div.find('h4').findall('a')
if len(alist) == 3:
update['author'] = (int(alist[-2].attrib['href'].split('/')[-1]),
to_unicode(alist[-2].text))
else:
m = re.match('Updated by (.*)', alist[0].tail.strip())
if m:
update['author'] = (0, to_unicode(m.group(1)))
update['date'] = self.parse_datetime(alist[-1].attrib['title'])
if div.find('div') is not None:
comment = div.find('div')
subdiv = comment.find('div')
if subdiv is not None:
# a subdiv which contains changes is found, move the tail text
# of this div to comment text, and remove it.
comment.text = (comment.text or '') + (subdiv.tail or '')
comment.remove(comment.find('div'))
update['message'] = self.parser.tostring(comment).strip()
else:
update['message'] = None
changes = []
try:
details = self.parser.select(div, 'ul.details', 1)
except BrokenPageError:
pass
else:
for li in details.findall('li'):
field = li.find('strong').text.decode('utf-8')
i = li.findall('i')
new = None
last = None
if len(i) > 0:
if len(i) == 2:
last = i[0].text.decode('utf-8')
new = i[-1].text.decode('utf-8')
elif li.find('strike') is not None:
last = li.find('strike').find('i').text.decode('utf-8')
elif li.find('a') is not None:
new = li.find('a').text.decode('utf-8')
else:
self.logger.warning('Unable to handle change for %s' % field)
changes.append((field, last, new))
update['changes'] = changes
params['updates'].append(update)
return params
class IssueLogTimePage(BasePage):
def logtime(self, hours, message):
self.browser.select_form(predicate=lambda form: form.attrs.get('action', '').endswith('/edit'))
self.browser['time_entry[hours]'] = '%.2f' % hours
self.browser['time_entry[comments]'] = message.encode('utf-8')
self.browser['time_entry[activity_id]'] = ['8']
self.browser.submit()
class IssueTimeEntriesPage(BasePage):
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/redmine/pages/wiki.py 0000664 0000000 0000000 00000002606 11666415431 0027742 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage
class WikiEditPage(BasePage):
def get_source(self):
return self.parser.select(self.document.getroot(), 'textarea#content_text', 1).text
def set_source(self, data, message):
self.browser.select_form(nr=1)
self.browser['content[text]'] = data.encode('utf-8')
if message:
self.browser['content[comments]'] = message.encode('utf-8')
self.browser.submit()
def get_authenticity_token(self):
wiki_form = self.parser.select(self.document.getroot(), 'form#wiki_form', 1)
return wiki_form.xpath('div/input')[0].get('value')
class WikiPage(BasePage):
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/sfr/ 0000775 0000000 0000000 00000000000 11666415431 0024471 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/sfr/__init__.py 0000664 0000000 0000000 00000000072 11666415431 0026601 0 ustar 00root root 0000000 0000000 from .backend import SfrBackend
__all__ = ['SfrBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/sfr/backend.py 0000664 0000000 0000000 00000004214 11666415431 0026433 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.messages import CantSendMessage, ICapMessages, ICapMessagesPost
from weboob.capabilities.account import ICapAccount, StatusField
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import Value, ValueBackendPassword
from .browser import SfrBrowser
__all__ = ['SfrBackend']
class SfrBackend(BaseBackend, ICapAccount, ICapMessages, ICapMessagesPost):
NAME = 'sfr'
MAINTAINER = 'Christophe Benz'
EMAIL = 'christophe.benz@gmail.com'
VERSION = '0.9.1'
DESCRIPTION = 'SFR french mobile phone provider'
LICENSE = 'AGPLv3+'
CONFIG = BackendConfig(Value('login', label='Login'),
ValueBackendPassword('password', label='Password'))
BROWSER = SfrBrowser
ACCOUNT_REGISTER_PROPERTIES = None
def create_default_browser(self):
return self.create_browser(self.config['login'].get(), self.config['password'].get())
# ICapMessagesPost methods
def get_account_status(self):
with self.browser:
return (StatusField('nb_remaining_free_sms', 'Number of remaining free SMS',
self.browser.get_nb_remaining_free_sms()),)
def post_message(self, message):
if not message.content.strip():
raise CantSendMessage(u'Message content is empty.')
with self.browser:
self.browser.post_message(message)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/sfr/browser.py 0000664 0000000 0000000 00000004601 11666415431 0026527 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import urllib
from .pages.compose import ClosePage, ComposePage, ConfirmPage, SentPage
from .pages.login import LoginPage
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
__all__ = ['SfrBrowser']
class SfrBrowser(BaseBrowser):
DOMAIN = 'www.sfr.fr'
PAGES = {
'http://messagerie-.+.sfr.fr/webmail/close_xms_tab.html': ClosePage,
'http://www.sfr.fr/xmscomposer/index.html\?todo=compose': ComposePage,
'http://www.sfr.fr/xmscomposer/mc/envoyer-texto-mms/confirm.html': ConfirmPage,
'https://www.sfr.fr/cas/login\?service=.*': LoginPage,
'http://www.sfr.fr/xmscomposer/mc/envoyer-texto-mms/send.html': SentPage,
}
def get_nb_remaining_free_sms(self):
if not self.is_on_page(ComposePage):
self.home()
return self.page.get_nb_remaining_free_sms()
def home(self):
self.location('http://www.sfr.fr/xmscomposer/index.html?todo=compose')
def is_logged(self):
return 'loginForm' not in [form.name for form in self.forms()]
def login(self):
service_url = 'http://www.sfr.fr/xmscomposer/j_spring_cas_security_check'
self.location('https://www.sfr.fr/cas/login?service=%s' % urllib.quote_plus(service_url), no_login=True)
self.page.login(self.username, self.password)
if not self.is_logged():
raise BrowserIncorrectPassword()
def post_message(self, message):
if not self.is_on_page(ComposePage):
self.home()
self.page.post_message(message)
if self.is_on_page(ConfirmPage):
self.page.confirm()
assert self.is_on_page(ClosePage) or self.is_on_page(SentPage)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/sfr/pages/ 0000775 0000000 0000000 00000000000 11666415431 0025570 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/sfr/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0027667 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/sfr/pages/compose.py 0000664 0000000 0000000 00000003567 11666415431 0027622 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from weboob.capabilities.messages import CantSendMessage
from weboob.tools.browser import BasePage
__all__ = ['ClosePage', 'ComposePage', 'ConfirmPage', 'SentPage']
class ClosePage(BasePage):
pass
class ComposePage(BasePage):
phone_regex = re.compile('^(\+33|0033|0)(6|7)(\d{8})$')
def get_nb_remaining_free_sms(self):
remaining_regex = re.compile(u'Il vous reste (?P.+) Texto gratuits vers les numéros SFR à envoyer aujourd\'hui')
text = self.parser.select(self.document.getroot(), '#smsReminder', 1).text.strip()
return remaining_regex.match(text).groupdict().get('nb')
def post_message(self, message):
receiver = message.thread.id
if self.phone_regex.match(receiver) is None:
raise CantSendMessage(u'Invalid receiver: %s' % receiver)
self.browser.select_form(nr=0)
self.browser['msisdns'] = receiver
self.browser['textMessage'] = message.content.encode('utf-8')
self.browser.submit()
class ConfirmPage(BasePage):
def confirm(self):
self.browser.select_form(nr=0)
self.browser.submit()
class SentPage(BasePage):
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/sfr/pages/login.py 0000664 0000000 0000000 00000002060 11666415431 0027250 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BasePage
__all__ = ['LoginPage']
class LoginPage(BasePage):
def login(self, login, password):
self.browser.select_form(nr=0)
self.browser['username'] = login
self.browser['password'] = password
self.browser['remember-me'] = ['on']
self.browser.submit()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/sfr/test.py 0000664 0000000 0000000 00000001543 11666415431 0026025 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class SFRTest(BackendTest):
BACKEND = 'sfr'
def test_sfr(self):
pass
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/simplyreadit/ 0000775 0000000 0000000 00000000000 11666415431 0026405 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/simplyreadit/__init__.py 0000664 0000000 0000000 00000001450 11666415431 0030516 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import SimplyreaditBackend
__all__ = ['SimplyreaditBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/simplyreadit/backend.py 0000664 0000000 0000000 00000002630 11666415431 0030347 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderBackend, DisplayPage
__all__ = ['SimplyreaditBackend']
class SimplyreaditBackend(GenericComicReaderBackend):
NAME = 'simplyreadit'
DESCRIPTION = 'Simplyreadit manga reading site'
BROWSER_PARAMS = dict(
img_src_xpath="//img[@class='open']/@src",
page_list_xpath="(//div[contains(@class,'dropdown_right')]/ul[@class='dropdown'])[1]/li/a/@href")
ID_TO_URL = 'http://www.simplyread.it/reader/read/%s'
ID_REGEXP = r'[^/]+(?:/[^/]+)*'
URL_REGEXP = r'.+simplyread.it/reader/read/(%s)/page/.+' % ID_REGEXP
PAGES = { r'http://.+\.simplyread.it/reader/read/.+': DisplayPage }
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/simplyreadit/test.py 0000664 0000000 0000000 00000001726 11666415431 0027744 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderTest
class SimplyreaditTest(GenericComicReaderTest):
BACKEND = 'simplyreadit'
def test_download(self):
return self._test_download('bonnouji/en/1/3')
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/societegenerale/ 0000775 0000000 0000000 00000000000 11666415431 0027035 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/societegenerale/__init__.py 0000664 0000000 0000000 00000001464 11666415431 0031153 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import SocieteGeneraleBackend
__all__ = ['SocieteGeneraleBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/societegenerale/backend.py 0000664 0000000 0000000 00000004072 11666415431 0031001 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
# python2.5 compatibility
from __future__ import with_statement
from weboob.capabilities.bank import ICapBank, AccountNotFound
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import ValueBackendPassword
from .browser import SocieteGenerale
__all__ = ['SocieteGeneraleBackend']
class SocieteGeneraleBackend(BaseBackend, ICapBank):
NAME = 'societegenerale'
MAINTAINER = 'Jocelyn Jaubert'
EMAIL = 'jocelyn.jaubert@gmail.com'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = u'Société Générale french bank\' website'
CONFIG = BackendConfig(ValueBackendPassword('login', label='Account ID', masked=False),
ValueBackendPassword('password', label='Password'))
BROWSER = SocieteGenerale
def create_default_browser(self):
return self.create_browser(self.config['login'].get(),
self.config['password'].get())
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
print _id
if not _id.isdigit():
raise AccountNotFound()
with self.browser:
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/societegenerale/browser.py 0000664 0000000 0000000 00000005570 11666415431 0031101 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from weboob.backends.societegenerale import pages
__all__ = ['SocieteGenerale']
class SocieteGenerale(BaseBrowser):
DOMAIN_LOGIN = 'particuliers.societegenerale.fr'
DOMAIN = 'particuliers.secure.societegenerale.fr'
PROTOCOL = 'https'
ENCODING = None # refer to the HTML encoding
PAGES = {
'https://particuliers.societegenerale.fr/.*': pages.LoginPage,
'.*restitution/cns_listeprestation.html': pages.AccountsList,
# '.*restitution/cns_detailCav.html.*': pages.AccountHistory,
}
def __init__(self, *args, **kwargs):
BaseBrowser.__init__(self, *args, **kwargs)
def home(self):
self.location('https://' + self.DOMAIN_LOGIN + '/index.html')
def is_logged(self):
return not self.is_on_page(pages.LoginPage)
def login(self):
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
assert self.password.isdigit()
if not self.is_on_page(pages.LoginPage):
self.location('https://' + self.DOMAIN_LOGIN + '/index.html')
self.page.login(self.username, self.password)
if self.is_on_page(pages.LoginPage):
raise BrowserIncorrectPassword()
def get_accounts_list(self):
if not self.is_on_page(pages.AccountsList):
self.location('/restitution/cns_listeprestation.html')
return self.page.get_list()
def get_account(self, id):
assert isinstance(id, basestring)
if not self.is_on_page(pages.AccountsList):
self.location('/restitution/cns_listeprestation.html')
l = self.page.get_list()
for a in l:
if a.id == id:
return a
return None
def get_history(self, account):
raise NotImplementedError()
if not self.is_on_page(pages.AccountHistory) or self.page.account.id != account.id:
self.location(account.link_id)
return self.page.get_operations()
def transfer(self, from_id, to_id, amount, reason=None):
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/societegenerale/captcha.py 0000664 0000000 0000000 00000006731 11666415431 0031021 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import hashlib
import Image
class TileError(Exception):
def __init__(self, msg, tile = None):
Exception.__init__(self, msg)
self.tile = tile
class Captcha:
def __init__(self, file, infos):
self.inim = Image.open(file)
self.infos = infos
self.nbr = int(infos["nblignes"])
self.nbc = int(infos["nbcolonnes"])
(self.nx,self.ny) = self.inim.size
self.inmat = self.inim.load()
self.map = {}
self.tiles = [[Tile(y * self.nbc + x) for y in xrange(4)] for x in xrange(4)]
def __getitem__(self, (x, y)):
return self.inmat[x % self.nx, y % self.ny]
def all_coords(self):
for y in xrange(self.ny):
for x in xrange(self.nx):
yield x, y
def get_codes(self, code):
s = ''
num = 0
for c in code:
index = self.map[int(c)].id
keycode = self.infos["keyCodes"][num * self.nbr * self.nbc + index]
s += keycode
if num < 5:
s += ','
num += 1
return s
def build_tiles(self):
for ty in xrange(0, self.nbc):
y = ty * 23
for tx in xrange(0, self.nbr):
x = tx * 24
tile = self.tiles[tx][ty]
for yy in xrange(y, y + 23):
for xx in xrange(x, x + 24):
tile.map.append(self[xx, yy])
num = tile.get_num()
if num > -1:
tile.valid = True
self.map[num] = tile
class Tile:
hash = {'ff1441b2c5f90703ef04e688e399aca5': 1,
'53d7f3dfd64f54723b231fc398b6be57': 2,
'5bcba7fa2107ba9a606e8d0131c162eb': 3,
'9db6e7ed063e5f9a69ab831e6cc0d721': 4,
'30ebb75bfa5077f41ccfb72e8c9cc15b': 5,
'61e27275e494038e524bc9fbbd0be130': 6,
'0e0816f1b743f320ca561f090df0fbb1': 7,
'11e7d4a6d447e66a5a112c1d9f7fc442': 8,
'2ea3c82768030d91571d360acf7a0f75': 9,
'28a834ebbf0238b46d3fffae1a0b781b': 0,
'04211db029ce488e07010f618a589c71': -1
}
def __init__(self, _id):
self.id = _id
self.valid = False
self.map = []
def __repr__(self):
return "" % (self.id, self.valid)
def checksum(self):
s = ''
for pxls in self.map:
for pxl in pxls:
s += '%02d' % pxl
return hashlib.md5(s).hexdigest()
def get_num(self):
sum = self.checksum()
try:
return self.hash[sum]
except KeyError:
self.display()
raise TileError('Tile not found ' + sum, self)
def display(self):
print self.checksum()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/societegenerale/pages/ 0000775 0000000 0000000 00000000000 11666415431 0030134 5 ustar 00root root 0000000 0000000 __init__.py 0000664 0000000 0000000 00000001625 11666415431 0032172 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/societegenerale/pages # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .accounts_list import AccountsList
from .login import LoginPage
class AccountPrelevement(AccountsList): pass
__all__ = ['LoginPage',
'AccountsList',
]
accounts_list.py 0000664 0000000 0000000 00000004123 11666415431 0033301 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/societegenerale/pages # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from weboob.capabilities.bank import Account
from weboob.tools.browser import BasePage
class AccountsList(BasePage):
LINKID_REGEXP = re.compile(".*ch4=(\w+).*")
def on_loaded(self):
pass
def get_list(self):
l = []
for tr in self.document.getiterator('tr'):
if tr.attrib.get('class', '') == 'LGNTableRow':
account = Account()
for td in tr.getiterator('td'):
if td.attrib.get('headers', '') == 'TypeCompte':
a = td.find('a')
account.label = a.text
account.link_id = a.get('href', '')
elif td.attrib.get('headers', '') == 'NumeroCompte':
id = td.text
id = id.replace(u'\xa0','')
account.id = id
elif td.attrib.get('headers', '') == 'Libelle':
pass
elif td.attrib.get('headers', '') == 'Solde':
balance = td.text
balance = balance.replace(u'\xa0','').replace(',','.')
if balance != "":
account.balance = float(balance)
else:
account.balance = 0.0
l.append(account)
return l
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/societegenerale/pages/login.py 0000664 0000000 0000000 00000005246 11666415431 0031625 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from logging import error
from weboob.tools.browser import BasePage, BrowserUnavailable
from weboob.backends.societegenerale.captcha import Captcha, TileError
from lxml import etree
__all__ = ['LoginPage']
class LoginPage(BasePage):
def on_loaded(self):
for td in self.document.getroot().cssselect('td.LibelleErreur'):
if td.text is None:
continue
msg = td.text.strip()
if 'indisponible' in msg:
raise BrowserUnavailable(msg)
def login(self, login, password):
DOMAIN_LOGIN = self.browser.DOMAIN_LOGIN
DOMAIN = self.browser.DOMAIN
url_login = 'https://' + DOMAIN_LOGIN + '/index.html'
base_url = 'https://' + DOMAIN
url = base_url + '/cvcsgenclavier?mode=jsom&estSession=0'
headers = {
'Referer': url_login
}
request = self.browser.request_class(url, None, headers)
infos_data = self.browser.readurl(request)
infos_xml = etree.XML(infos_data)
infos = {}
for el in ("cryptogramme", "nblignes", "nbcolonnes"):
infos[el] = infos_xml.find(el).text
infos["grille"] = ""
for g in infos_xml.findall("grille"):
infos["grille"] += g.text + ","
infos["keyCodes"] = infos["grille"].split(",")
url = base_url + '/cvcsgenimage?modeClavier=0&cryptogramme=' + infos["cryptogramme"]
img = Captcha(self.browser.openurl(url), infos)
try:
img.build_tiles()
except TileError, err:
error("Error: %s" % err)
if err.tile:
err.tile.display()
self.browser.openurl(url_login)
self.browser.select_form('authentification')
self.browser.set_all_readonly(False)
self.browser['codcli'] = login
self.browser['codsec'] = img.get_codes(password)
self.browser['cryptocvcs'] = infos["cryptogramme"]
self.browser.submit()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/societegenerale/test.py 0000664 0000000 0000000 00000002066 11666415431 0030372 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class SocieteGeneraleTest(BackendTest):
BACKEND = 'societegenerale'
def test_societegenerale(self):
l = list(self.backend.iter_accounts())
if len(l) > 0:
a = l[0]
list(self.backend.iter_operations(a))
list(self.backend.iter_history(a))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/transilien/ 0000775 0000000 0000000 00000000000 11666415431 0026047 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/transilien/__init__.py 0000664 0000000 0000000 00000001470 11666415431 0030162 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Hébert, Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import TransilienBackend
__all__ = ['TransilienBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/transilien/backend.py 0000664 0000000 0000000 00000004713 11666415431 0030015 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Hébert, Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.travel import ICapTravel, Station, Departure, RoadStep
from weboob.tools.backend import BaseBackend
from .browser import Transilien
from .stations import STATIONS
class TransilienBackend(BaseBackend, ICapTravel):
NAME = 'transilien'
MAINTAINER = u'Julien Hébert'
EMAIL = 'juke@free.fr'
VERSION = '0.9.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = "Transports in Paris"
BROWSER = Transilien
def iter_station_search(self, pattern):
pattern = pattern.lower()
for _id, name in STATIONS.iteritems():
if name.lower().find(pattern) >= 0:
yield Station(_id, name)
def iter_station_departures(self, station_id, arrival_id=None):
with self.browser:
for i, d in enumerate(self.browser.iter_station_departures(station_id, arrival_id)):
departure = Departure(i, d['type'], d['time'])
departure.departure_station = d['departure']
departure.arrival_station = d['arrival']
departure.late = d['late']
departure.information = d['late_reason']
departure.plateform = d['plateform']
yield departure
def iter_roadmap(self, departure, arrival, filters):
with self.browser:
roadmap = self.browser.get_roadmap(departure, arrival, filters)
for s in roadmap['steps']:
step = RoadStep(s['id'])
step.line = s['line']
step.start_time = s['start_time']
step.end_time = s['end_time']
step.departure = s['departure']
step.arrival = s['arrival']
step.duration = s['duration']
yield step
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/transilien/browser.py 0000664 0000000 0000000 00000005601 11666415431 0030106 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Hébert, Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser, BasePage, BrowserUnavailable
from .pages.departures import DeparturesPage
from .pages.roadmap import RoadmapSearchPage, RoadmapConfirmPage, RoadmapPage
class UnavailablePage(BasePage):
def on_loaded(self):
raise BrowserUnavailable('Website is currently unavailable')
class Transilien(BaseBrowser):
DOMAIN = 'www.transilien.com'
PROTOCOL = 'https'
USER_AGENT = BaseBrowser.USER_AGENTS['microb']
PAGES = {'https://www\.transilien\.com/web/ITProchainsTrainsAvecDest\.do\?.*': DeparturesPage,
'https://www\.transilien\.com/web/ITProchainsTrains\.do\?.*': DeparturesPage,
'https://www\.transilien\.com/web/site.*': RoadmapSearchPage,
'https://www\.transilien\.com/web/RedirectHI.do.*': RoadmapConfirmPage,
'https://www\.transilien\.com/web/RedirectHIIntermediaire.do.*': RoadmapPage,
'https://www\.transilien\.com/transilien_sncf_maintenance_en_cours.htm': UnavailablePage,
}
def is_logged(self):
""" Do not need to be logged """
return True
def iter_station_search(self, pattern):
pass
def iter_station_departures(self, station_id, arrival_id=None):
if arrival_id:
self.location('https://www.transilien.com/web/ITProchainsTrainsAvecDest.do?codeTr3aDepart=%s&codeTr3aDest=%s&urlModule=/site/pid/184&gareAcc=true' % (station_id, arrival_id))
else:
self.location('https://www.transilien.com/web/ITProchainsTrains.do?tr3a=%s&urlModule=/site/pid/184' % station_id)
return self.page.iter_routes()
def get_roadmap(self, departure, arrival, filters):
self.location('/web/site/accueil/etat-trafic/chercher-itineraire/lang/en')
assert self.is_on_page(RoadmapSearchPage)
self.page.search(departure, arrival, filters.departure_time, filters.arrival_time)
assert self.is_on_page(RoadmapConfirmPage)
self.page.confirm()
assert self.is_on_page(RoadmapPage)
roadmap = {}
roadmap['steps'] = list(self.page.get_steps())
return roadmap
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/transilien/pages/ 0000775 0000000 0000000 00000000000 11666415431 0027146 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/transilien/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0031245 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/transilien/pages/departures.py 0000664 0000000 0000000 00000004503 11666415431 0031700 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Hébert, Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import datetime
from weboob.tools.misc import to_unicode
from weboob.tools.browser import BasePage, BrokenPageError
__all__ = ['StationNotFound', 'DeparturesPage']
class StationNotFound(Exception):
pass
class DeparturesPage(BasePage):
def iter_routes(self):
try:
table = self.parser.select(self.document.getroot(), 'table.horaires3', 1)
except BrokenPageError:
raise StationNotFound('Station not found')
departure = self.parser.select(table, 'td.caption strong', 1).text
for tr in table.findall('tr'):
if len(tr.findall('td')) != 4:
continue
code_mission = self.parser.select(tr, 'td[headers=Code_de_mission] a', 1).text.strip()
time = self.parser.select(tr, 'td[headers=Heure_de_passage]', 1).text.strip()
destination = self.parser.select(tr, 'td[headers=Destination]', 1).text.strip()
plateform = self.parser.select(tr, 'td[headers=Voie]', 1).text.strip()
try :
time = datetime.datetime.combine(datetime.date.today(), datetime.time(*[int(x) for x in time.split(':')]))
except ValueError:
self.logger.warning('Unable to parse datetime "%s"' % time)
yield {'type': to_unicode(code_mission),
'time': time,
'departure': to_unicode(departure),
'arrival': to_unicode(destination),
'late': datetime.time(),
'late_reason': None,
'plateform': to_unicode(plateform)}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/transilien/pages/roadmap.py 0000664 0000000 0000000 00000011003 11666415431 0031136 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
import datetime
from weboob.capabilities.travel import RoadmapError
from weboob.tools.browser import BasePage
from weboob.tools.misc import to_unicode
from weboob.tools.mech import ClientForm
__all__ = ['RoadmapPage']
class RoadmapSearchPage(BasePage):
def search(self, departure, arrival, departure_time, arrival_time):
self.browser.select_form('formHiRecherche')
self.browser['lieuDepart'] = departure.encode('utf-8')
self.browser['lieuArrivee'] = arrival.encode('utf-8')
time = None
if departure_time:
self.browser['typeHeure'] = ['1']
time = departure_time
elif arrival_time:
self.browser['typeHeure'] = ['-1']
time = arrival_time
if time:
try:
self.browser['jour'] = ['%d' % time.day]
self.browser['mois'] = ['%02d/%d' % (time.month, time.year)]
self.browser['heure'] = ['%02d' % time.hour]
self.browser['minutes'] = ['%02d' % (time.minute - (time.minute%5))]
except ClientForm.ItemNotFoundError:
raise RoadmapError('Unable to establish a roadmap with %s time at "%s"' % ('departure' if departure_time else 'arrival', time))
self.browser.submit()
class RoadmapConfirmPage(BasePage):
def select(self, name, num):
try:
self.browser[name] = str(num)
except TypeError:
self.browser[name] = [str(num)]
def confirm(self):
self.browser.select_form('form1')
self.browser.set_all_readonly(False)
self.select('idDepart', 1)
self.select('idArrivee', 1)
self.browser['modeTransport'] = ['0']
self.browser['trainRer'] = 'true'
self.browser['bus'] = 'false'
self.browser['tramway'] = 'true'
self.browser['bateau'] = 'false'
self.browser.submit()
class RoadmapPage(BasePage):
def get_steps(self):
errors = []
for p in self.parser.select(self.document.getroot(), 'p.errors'):
if p.text:
errors.append(p.text.strip())
if len(errors) > 0:
raise RoadmapError('Unable to establish a roadmap: %s' % ', '.join(errors))
current_step = None
i = 0
for tr in self.parser.select(self.document.getroot(), 'table.horaires2 tbody tr'):
if not 'class' in tr.attrib:
continue
elif tr.attrib['class'] == 'trHautTroncon':
current_step = {}
current_step['id'] = i
i += 1
current_step['start_time'] = self.parse_time(self.parser.select(tr, 'td.formattedHeureDepart p', 1).text.strip())
current_step['line'] = self.parser.select(tr, 'td.rechercheResultatColumnMode img')[-1].attrib['alt']
current_step['departure'] = to_unicode(self.parser.select(tr, 'td.descDepart p strong', 1).text.strip())
current_step['duration'] = self.parse_duration(self.parser.select(tr, 'td.rechercheResultatVertAlign', 1).text.strip())
elif tr.attrib['class'] == 'trBasTroncon':
current_step['end_time'] = self.parse_time(self.parser.select(tr, 'td.formattedHeureArrivee p', 1).text.strip())
current_step['arrival'] = to_unicode(self.parser.select(tr, 'td.descArrivee p strong', 1).text.strip())
yield current_step
def parse_time(self, time):
h, m = time.split('h')
return datetime.time(int(h), int(m))
def parse_duration(self, dur):
m = re.match('(\d+)min.', dur)
if m:
return datetime.timedelta(minutes=int(m.group(1)))
m = re.match('(\d+)h(\d+)', dur)
if m:
return datetime.timedelta(hours=int(m.group(1)),
minutes=int(m.group(2)))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/transilien/pages/station.py 0000664 0000000 0000000 00000001341 11666415431 0031200 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/transilien/stations.py 0000664 0000000 0000000 00000024540 11666415431 0030272 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Hébert, Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
STATIONS = {
'BXN': u'BAGNEAUX SUR LOING',
'BJR': u'BOIS LE ROI',
'BOM': u'BOURRON MARLOTTE GREZ',
'BXI': u'BOUSSY ST ANTOINE',
'BNY': u'BRUNOY',
'CES': u'CESSON',
'CLY': u'CHANTILLY GOUVIEUX',
'CLX': u'CHATELET LES HALLES',
'CBV': u'COMBS LA VILLE QUINCY',
'COE': u'CORBEIL ESSONNES',
'CL': u'CREIL',
'DDI': u'DORDIVES',
'EVR': u'EVRY',
'EVC': u'EVRY COURCOURONNES',
'FFY': u'FERRIERES FONTENAY',
'FON': u'FONTAINEBLEAU AVON',
'GDS': u'GARE DU NORD',
'GAJ': u'GARGES SARCELLES',
'GOU': u'GOUSSAINVILLE',
'GBG': u'GRAND BOURG',
'GGG': u'GRIGNY CENTRE',
'JY': u'JUVISY',
'BBN': u'LA BORNE BLANCHE',
'BFX': u'LE BRAS DE FER',
'WEE': u'LE MEE',
'VD': u'LE VERT DE MAISONS',
'LNX': u'LES NOUES',
'LIU': u'LIEUSAINT MOISSY',
'LOV': u'LOUVRES',
'MFA': u'MAISONS ALFORT ALFORTVILLE',
'MEL': u'MELUN',
'MS': u'MONTARGIS',
'MTU': u'MONTEREAU',
'KRW': u'MONTGERON CROSNE',
'MKN': u'MONTIGNY SUR LOING',
'MOR': u'MORET VENEUX LES SABLONS',
'NSP': u'NEMOURS ST PIERRE',
'OBP': u'ORANGIS BOIS DE L EPINE',
'ORY': u'ORRY LA VILLE COYE LA FORET',
'PRF': u'PIERREFITTE STAINS',
'RIS': u'RIS ORANGIS',
'ZTN': u'SAVIGNY LE TEMPLE NANDY',
'SPP': u'SOUPPES CHÂTEAU LANDON',
'SDE': u'ST DENIS',
'SF': u'ST MAMMES',
'SFD': u'STADE DE FRANCE ST DENIS',
'SUR': u'SURVILLIERS FOSSES',
'TMR': u'THOMERY',
'VGS': u'VIGNEUX SUR SEINE',
'VP': u'VILLENEUVE PRAIRIE',
'VSG': u'VILLENEUVE ST GEORGES',
'VTV': u'VILLENEUVE TRIAGE',
'VIB': u'VILLIERS LE BEL GONESSE ARNOUVILLE',
'VWC': u'VIRY CHATILLON',
'YES': u'YERRES',
'ARW': u'ARGENTEUIL',
'AEH': u'AUBERGENVILLE ELISABETHVILLE',
'BEC': u'BECON LES BRUYERES',
'BCO': u'BOIS COLOMBES',
'CWJ': u'CHAVILLE RIVE DROITE',
'CLL': u'CLICHY LEVALLOIS',
'CBK': u'COLOMBES',
'CSH': u'CONFLANS SAINTE HONORINE',
'CPA': u'CORMEILLES EN PARISIS',
'KOU': u'COURBEVOIE',
'EPO': u'EPONE MEZIERES',
'ERA': u'ERAGNY NEUVILLE',
'ERE': u'ERMONT EAUBONNE',
'PSL': u'GARE ST LAZARE',
'HRY': u'HERBLAY',
'HAR': u'HOUILLES CARRIERES SUR SEINE',
'LDU': u'LA DEFENSE GARE SNCF',
'FMY': u'LA FRETTE MONTIGNY',
'LGK': u'LA GARENNE COLOMBES',
'LSD': u'LE STADE',
'VDO': u'LE VAL D OR',
'KVE': u'LES CLAIRIERES DE VERNEUIL',
'LMU': u'LES MUREAUX',
'LWA': u'LES VALLEES',
'MLF': u'MAISONS LAFFITTE',
'MTE': u'MANTES LA JOLIE',
'MTQ': u'MANTES STATION',
'MFL': u'MONTREUIL',
'NUN': u'NANTERRE UNIVERSITE',
'PSY': u'POISSY',
'PTC': u'PONT CARDINET',
'PSE': u'PONTOISE',
'PTX': u'PUTEAUX',
'SNN': u'SANNOIS',
'SVL': u'SARTROUVILLE',
'VDV': u'SEVRES VILLE D AVRAY',
'SCD': u'ST CLOUD',
'XOA': u'ST OUEN L AUMONE QUARTIER DE L EGLIS',
'MVH': u'SURESNES MONT VALERIEN',
'VDA': u'VAL D ARGENTEUIL',
'VET': u'VERNOUILLET VERNEUIL',
'VRD': u'VERSAILLES RIVE DROITE',
'VSW': u'VILLENNES SUR SEINE',
'VFD': u'VIROFLAY RIVE DROITE',
'ABL': u'ABLON',
'ARP': u'ARPAJON',
'ATH': u'ATHIS MONS',
'BFM': u'BIBLIOTHEQUE F. MITTERRAND',
'BIS': u'BIEVRES',
'BVI': u'BOULEVARD VICTOR',
'BY': u'BRETIGNY',
'BIH': u'BREUILLET BRUYERES LE CHATEL',
'BRW': u'BREUILLET VILLAGE',
'CPM': u'CHAMP DE MARS TOUR EIFFEL',
'CHV': u'CHAVILLE VELIZY',
'CAZ': u'CHILLY MAZARIN',
'CLR': u'CHOISY LE ROI',
'D': u'DOURDAN',
'ELY': u'EGLY',
'EYO': u'EPINAY SUR ORGE',
'GBI': u'GRAVIGNY BALIZY',
'IGY': u'IGNY',
'INV': u'INVALIDES',
'ISY': u'ISSY',
'ISP': u'ISSY VAL DE SEINE',
'IV': u'IVRY SUR SEINE',
'JVL': u'JAVEL',
'JAS': u'JOUY EN JOSAS',
'NG': u'LA NORVILLE ST GERMAIN LES ARPAJON',
'LAD': u'LES ARDOINES',
'LJU': u'LONGJUMEAU',
'MPU': u'MASSY PALAISEAU RER C',
'MFY': u'MEUDON VAL FLEURY',
'MDS': u'MUSEE D ORSAY',
'PJ': u'PETIT JOUY LES LOGES',
'PV': u'PETIT VAUX',
'PDM': u'PONT DE L ALMA',
'POA': u'PORCHEFONTAINE',
'SAO': u'SAVIGNY SUR ORGE',
'SXE': u'SERMAISE',
'SCW': u'ST CHERON',
'SHL': u'ST MICHEL NOTRE DAME',
'SHO': u'ST MICHEL SUR ORGE',
'SXG': u'STE GENEVIEVE DES BOIS',
'VBO': u'VAUBOYEN',
'VC': u'VERSAILLES CHANTIERS',
'VRG': u'VERSAILLES R G CHATEAU DE VERSAILLES',
'VRI': u'VILLENEUVE LE ROI',
'VFG': u'VIROFLAY RIVE GAUCHE',
'VY': u'VITRY SUR SEINE',
'BSO': u'BOURAY',
'CHK': u'CHAMARANDE',
'ETP': u'ETAMPES',
'ETY': u'ETRECHY',
'PZB': u'GARE D\'AUSTERLITZ',
'LYO': u'LARDY',
'MSX': u'MAROLLES EN HUREPOIX',
'SME': u'ST MARTIN D ETAMPES',
'CME': u'CHAMPAGNE SUR SEINE',
'CJR': u'CHARTRETTES',
'HER': u'HERICY',
'GPA': u'LA GRANDE PAROISSE',
'LYQ': u'LIVRY SUR SEINE',
'VSS': u'VERNOU SUR SEINE',
'VUN': u'VULAINES SUR SEINE SAMOREAU',
'PLY': u'GARE DE LYON',
'FPO': u'FONTAINE LE PORT',
'CJN': u'CHANGIS ST JEAN',
'CTH': u'CHATEAU THIERRY',
'CSG': u'CHELLES GOURNAY',
'CYZ': u'CHEZY SUR MARNE',
'CO': u'COULOMMIERS',
'EY': u'ESBLY',
'FMP': u'FAREMOUTIERS POMMEUSE',
'GCM': u'GUERARD LA CELLE SUR MORIN',
'LFJ': u'LA FERTE SOUS JOUARRE',
'LGY': u'LAGNY THORIGNY',
'MLB': u'MARLES EN BRIE',
'MEA': u'MEAUX',
'MOF': u'MORTCERF',
'MXK': u'MOUROUX',
'NAU': u'NANTEUIL SAACY',
'NAA': u'NOGENT L ARTAUD CHARLY',
'TOU': u'TOURNAN',
'TLP': u'TRILPORT',
'VAI': u'VAIRES TORCY',
'AEE': u'ASNIERES',
'AUU': u'AUBER',
'BQA': u'BRY SUR MARNE',
'BXG': u'BUSSY ST GEORGES',
'CGP': u'CHARLES DE GAULLE-ETOILE',
'GYN': u'GARE DE LYON',
'GAW': u'LA DEFENSE RER A',
'LQN': u'LOGNES',
'MVC': u'MARNE LA VALLEE CHESSY',
'NAF': u'NANTERRE PREFECTURE SNCF',
'NTN': u'NATION',
'NYP': u'NEUILLY PLAISANCE',
'NSL': u'NOISIEL',
'NYC': u'NOISY CHAMPS',
'NYG': u'NOISY LE GRAND MONT D EST',
'TOC': u'TORCY MARNE LA VALLEE',
'VDE': u'VAL D EUROPE',
'VFR': u'VAL DE FONTENAY RER A',
'VNC': u'VINCENNES',
'CJV': u'CERGY LE HAUT',
'CYP': u'CERGY PREFECTURE',
'CYC': u'CERGY ST CHRISTOPHE',
'CFD': u'CONFLANS FIN D OISE',
'NUE': u'NEUVILLE UNIVERSITE',
'RYR': u'AEROPORT CDG 2 TGV',
'ATW': u'ANTONY',
'ALC': u'AUBERVILLIERS LA COURNEUVE',
'AB': u'AULNAY SOUS BOIS',
'BAM': u'BLANC MESNIL',
'BQQ': u'BOURG LA REINE',
'BVJ': u'BURES SUR YVETTE',
'CUF': u'CITE UNIVERSITAIRE',
'CVW': u'COURCELLE SUR YVETTE',
'DFR': u'DENFERT ROCHEREAU',
'DRN': u'DRANCY',
'FMN': u'FONTAINE MICHALON',
'GIF': u'GIF SUR YVETTE',
'XBY': u'LA CROIX DE BERNY',
'HAQ': u'LA HACQUINIERE',
'LPN': u'LA PLAINE STADE DE FRANCE',
'LBT': u'LE BOURGET',
'GUW': u'LE GUICHET',
'BQC': u'LES BACONNETS',
'LZV': u'LOZERE',
'LXJ': u'LUXEMBOURG',
'MP': u'MASSY PALAISEAU RER B',
'MVP': u'MASSY VERRIERES RER B',
'ORS': u'ORSAY VILLE',
'PAX': u'PALAISEAU',
'PAW': u'PALAISEAU VILLEBON',
'PCX': u'PARC DE SCEAUX',
'PEX': u'PARC DES EXPOSITIONS',
'PWR': u'PORT ROYAL',
'BDE': u'SEVRAN BEAUDOTTES',
'XND': u'ST MICHEL NOTRE DAME',
'SNM': u'ST REMY LES CHEVREUSE',
'VPN': u'VILLEPINTE',
'RSY': u'AEROPORT CDG 1',
'ARK': u'ARCUEIL CACHAN',
'BGK': u'BAGNEUX',
'GTL': u'GENTILLY',
'LJA': u'LAPLACE',
'CVF': u'CHANTELOUP LES VIGNES',
'GGV': u'GARGENVILLE',
'IPO': u'ISSOU PORCHEVILLE',
'JUZ': u'JUZIERS',
'LIM': u'LIMAY',
'MHD': u'MEULAN HARDRICOURT',
'TPA': u'THUN LE PARADIS',
'TSS': u'TRIEL SUR SEINE',
'VXS': u'VAUX SUR SEINE',
'FNR': u'FONTENAY AUX ROSES',
'MY': u'MITRY CLAYE',
'RNS': u'ROBINSON',
'SKX': u'SCEAUX',
'SEV': u'SEVRAN LIVRY',
'VGL': u'VERT GALANT',
'VII': u'VILLEPARISIS MITRY LE NEUF',
'BOF': u'BOUFFEMONT MOISSELLES',
'DEU': u'DEUIL MONTMAGNY',
'DMO': u'DOMONT',
'ECZ': u'ECOUEN EZANVILLE',
'EPV': u'EPINAY VILLETANEUSE',
'PNB': u'GARE DU NORD',
'GRL': u'GROSLAY',
'LUZ': u'LUZARCHES',
'MSO': u'MONTSOULT MAFFLIERS',
'SLL': u'SARCELLES ST BRICE',
'SWY': u'SEUGY',
'VMS': u'VIARMES',
'VW': u'VILLAINES',
'CH': u'CHARTRES',
'CVI': u'CHAVILLE RIVE GAUCHE',
'CMA': u'CLAMART',
'CGW': u'COIGNIERES',
'DX': u'DREUX',
'EPN': u'EPERNON',
'FAF': u'FONTENAY LE FLEURY',
'GAQ': u'GARANCIERES LA QUEUE',
'GZA': u'GAZERAN',
'HOA': u'HOUDAN',
'JOY': u'JOUY',
'VYL': u'LA VERRIERE',
'LPE': u'LE PERRAY',
'LSI': u'LES ESSARTS LE ROI',
'MTN': u'MAINTENON',
'MBR': u'MARCHEZAIS BROUE',
'MDN': u'MEUDON',
'MLM': u'MONTFORT L AMAURY MERE',
'OGB': u'ORGERUS BEHOUST',
'PMP': u'PARIS MONTPARNASSE',
'PG': u'PLAISIR GRIGNON',
'PIE': u'PLAISIR LES CLAYES',
'RBT': u'RAMBOUILLET',
'SVR': u'SEVRES RIVE GAUCHE',
'SCR': u'ST CYR',
'SPI': u'ST PIAT',
'SQY': u'ST QUENTIN EN YVELINES',
'TAE': u'TACOIGNIERES RICHEBOURG',
'TVO': u'TRAPPES',
'VMK': u'VANVES MALAKOFF',
'VEP': u'VILLEPREUX LES CLAYES',
'VEH': u'VILLIERS NEAUPHLE PONTCHARTRAIN',
'CEG': u'CHAMP DE COURSES D ENGHIEN',
'CPO': u'CHAMPAGNE SUR OISE',
'EN': u'ENGHIEN LES BAINS',
'ERT': u'ERMONT EAUBONNE',
'ERM': u'ERMONT HALTE',
'FPN': u'FREPILLON',
'GNX': u'GROS NOYER ST PRIX',
'IAP': u'L ISLE ADAM PARMAIN',
'LBJ': u'LA BARRE ORMESSON',
'MLV': u'MERIEL',
'MWO': u'MERY SUR OISE',
'PEB': u'PERSAN BEAUMONT',
'SLF': u'ST LEU LA FORET',
'TVY': u'TAVERNY',
'VMD': u'VALMONDOIS',
'VCX': u'VAUCELLES',
'MJM': u'MAREIL SUR MAULDRE',
'MAE': u'MAULE',
'NZL': u'NEZEL AULNAY',
'PAA': u'GARE DE LYON',
}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/transilien/test.py 0000664 0000000 0000000 00000003002 11666415431 0027373 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import datetime
from weboob.capabilities.travel import RoadmapFilters
from weboob.tools.test import BackendTest
class TransilienTest(BackendTest):
BACKEND = 'transilien'
def test_departures(self):
stations = list(self.backend.iter_station_search('defense'))
self.assertTrue(len(stations) > 0)
list(self.backend.iter_station_departures(stations[0].id))
def test_roadmap(self):
filters = RoadmapFilters()
roadmap = list(self.backend.iter_roadmap('Puteaux', u'École Militaire', filters))
self.assertTrue(len(roadmap) > 0)
filters.arrival_time = datetime.datetime.now() + datetime.timedelta(days=1)
roadmap = list(self.backend.iter_roadmap('Puteaux', u'Aulnay-sous-Bois', filters))
self.assertTrue(len(roadmap) > 0)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/yahoo/ 0000775 0000000 0000000 00000000000 11666415431 0025016 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/yahoo/__init__.py 0000664 0000000 0000000 00000001436 11666415431 0027133 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import YahooBackend
__all__ = ['YahooBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/yahoo/backend.py 0000664 0000000 0000000 00000010332 11666415431 0026756 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
import urllib2
from xml.dom import minidom
# TODO store datetime objects instead of strings
# from datetime import datetime
from weboob.capabilities.weather import ICapWeather, CityNotFound, Current, Forecast, City
from weboob.tools.backend import BaseBackend
from weboob.tools.browser import BaseBrowser
__all__ = ['YahooBackend']
class YahooBackend(BaseBackend, ICapWeather):
NAME = 'yahoo'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
DESCRIPTION = 'Yahoo'
LICENSE = 'AGPLv3+'
BROWSER = BaseBrowser
WEATHER_URL = 'http://weather.yahooapis.com/forecastrss?w=%s&u=%s'
SEARCH_URL = 'http://fr.meteo.yahoo.com/search/weather?p=%s'
def create_default_browser(self):
return self.create_browser()
def iter_city_search(self, pattern):
# minidom doesn't seem to work with that page
#handler = urllib2.urlopen((self.SEARCH_URL % pattern).replace(' ','+'))
#dom = minidom.parse(handler)
#handler.close()
#results = dom.getElementById('search-results')
#for no in results.childNodes:
# print no.nodeValue
# so i use a basic but efficient parsing
with self.browser:
content = self.browser.readurl((self.SEARCH_URL % pattern.encode('utf-8')).replace(' ','+'))
page=''
for line in content.split('\n'):
if "" in line and "Prévisions et Temps" in line:
page="direct"
elif "" in line and "Résultats de la recherche" in line:
page="resultats"
if page == "resultats":
if '/redirwoei/' in line:
cities = line.split('/redirwoei/')
for c in cities:
if "strong" in c:
cid = c.split("'")[0]
cname = c.split("'")[1].replace(">","").replace("","").split("")[0]
yield City(cid, cname.decode('utf-8'))
elif page == "direct":
if 'div id="yw-breadcrumb"' in line:
l = line.split('')
region = l[2].split('>')[-1]
country = l[1].split('>')[-1]
city = l[3].split('
')[1].replace('
','')
cid = line.split("/?unit")[0].split('-')[-1]
yield City(cid, (city+", "+region+", "+country).decode('utf-8'))
def _get_weather_dom(self, city_id):
handler = urllib2.urlopen(self.WEATHER_URL % (city_id, 'c'))
dom = minidom.parse(handler)
handler.close()
if not dom.getElementsByTagName('yweather:condition'):
raise CityNotFound('City not found: %s' % city_id)
return dom
def get_current(self, city_id):
dom = self._get_weather_dom(city_id)
current = dom.getElementsByTagName('yweather:condition')[0]
return Current(current.getAttribute('date'), int(current.getAttribute('temp')), current.getAttribute('text'), 'C')
def iter_forecast(self, city_id):
dom = self._get_weather_dom(city_id)
for forecast in dom.getElementsByTagName('yweather:forecast'):
yield Forecast(forecast.getAttribute('date'),
int(forecast.getAttribute('low')),
int(forecast.getAttribute('high')),
forecast.getAttribute('text'),
'C',
)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/yahoo/test.py 0000664 0000000 0000000 00000002251 11666415431 0026347 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class YahooTest(BackendTest):
BACKEND = 'yahoo'
def test_meteo(self):
l = list(self.backend.iter_city_search('paris'))
self.assertTrue(len(l) > 0)
city = l[0]
current = self.backend.get_current(city.id)
self.assertTrue(current.temp > -20 and current.temp < 50)
forecasts = list(self.backend.iter_forecast(city.id))
self.assertTrue(len(forecasts) > 0)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youjizz/ 0000775 0000000 0000000 00000000000 11666415431 0025422 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youjizz/__init__.py 0000664 0000000 0000000 00000001443 11666415431 0027535 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import YoujizzBackend
__all__ = ['YoujizzBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youjizz/backend.py 0000664 0000000 0000000 00000004106 11666415431 0027364 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.video import ICapVideo
from weboob.tools.backend import BaseBackend
from .browser import YoujizzBrowser
from .video import YoujizzVideo
__all__ = ['YoujizzBackend']
class YoujizzBackend(BaseBackend, ICapVideo):
NAME = 'youjizz'
MAINTAINER = 'Roger Philibert'
EMAIL = 'roger.philibert@gmail.com'
VERSION = '0.9.1'
DESCRIPTION = 'Youjizz videos website'
LICENSE = 'AGPLv3+'
BROWSER = YoujizzBrowser
def get_video(self, _id):
with self.browser:
video = self.browser.get_video(_id)
return video
def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
if not nsfw:
return set()
with self.browser:
return self.browser.iter_search_results(pattern)
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
with self.browser:
video = self.browser.get_video(YoujizzVideo.id2url(video.id), video)
if 'thumbnail' in fields and video.thumbnail:
with self.browser:
video.thumbnail.data = self.browser.readurl(video.thumbnail.url)
return video
OBJECTS = {YoujizzVideo: fill_video}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youjizz/browser.py 0000664 0000000 0000000 00000003463 11666415431 0027465 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import urllib
from weboob.tools.browser import BaseBrowser
from weboob.tools.browser.decorators import id2url
from .pages.index import IndexPage
from .pages.video import VideoPage
from .video import YoujizzVideo
__all__ = ['YoujizzBrowser']
class YoujizzBrowser(BaseBrowser):
DOMAIN = 'youjizz.com'
ENCODING = None
PAGES = {r'http://.*youjizz\.com/?': IndexPage,
r'http://.*youjizz\.com/index.php': IndexPage,
r'http://.*youjizz\.com/search/(?P.+)\.html': IndexPage,
r'http://.*youjizz\.com/videos/(?P.+)\.html': VideoPage,
}
@id2url(YoujizzVideo.id2url)
def get_video(self, url, video=None):
self.location(url)
assert self.is_on_page(VideoPage), 'Should be on video page.'
return self.page.get_video(video)
def iter_search_results(self, pattern):
if not pattern:
self.home()
else:
self.location('/search/%s-1.html' % (urllib.quote_plus(pattern.encode('utf-8'))))
assert self.is_on_page(IndexPage)
return self.page.iter_videos()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youjizz/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026521 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youjizz/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0030620 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youjizz/pages/index.py 0000664 0000000 0000000 00000004064 11666415431 0030206 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import datetime
import re
from weboob.tools.browser import BasePage
from weboob.tools.browser import BrokenPageError
from ..video import YoujizzVideo
__all__ = ['IndexPage']
class IndexPage(BasePage):
def iter_videos(self):
span_list = self.parser.select(self.document.getroot(), 'span#miniatura')
for span in span_list:
a = self.parser.select(span, 'a', 1)
url = a.attrib['href']
_id = re.sub(r'/videos/(.+)\.html', r'\1', url)
thumbnail_url = span.find('.//img').attrib['src']
title_el = self.parser.select(span, 'span#title1', 1)
title = title_el.text.strip()
time_span = self.parser.select(span, 'span.thumbtime span', 1)
time_txt = time_span.text.strip().replace(';', ':')
if time_txt == 'N/A':
minutes, seconds = 0, 0
elif ':' in time_txt:
minutes, seconds = (int(v) for v in time_txt.split(':'))
else:
raise BrokenPageError('Unable to parse the video duration: %s' % time_txt)
yield YoujizzVideo(_id,
title=title,
duration=datetime.timedelta(minutes=minutes, seconds=seconds),
thumbnail_url=thumbnail_url,
)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youjizz/pages/video.py 0000664 0000000 0000000 00000004313 11666415431 0030202 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import datetime
import lxml.html
import re
from weboob.capabilities.base import NotAvailable
from weboob.tools.browser import BasePage, BrokenPageError
from weboob.tools.misc import to_unicode
from ..video import YoujizzVideo
__all__ = ['VideoPage']
class VideoPage(BasePage):
def get_video(self, video=None):
_id = to_unicode(self.group_dict['id'])
if video is None:
video = YoujizzVideo(_id)
title_el = self.parser.select(self.document.getroot(), 'title', 1)
video.title = to_unicode(title_el.text.strip())
# youjizz HTML is crap, we must parse it with regexps
data = lxml.html.tostring(self.document.getroot())
m = re.search(r'.*?Runtime.*? (.+?)', data)
if m:
txt = m.group(1).strip()
if txt == 'Unknown':
video.duration = NotAvailable
else:
minutes, seconds = (int(v) for v in to_unicode(txt).split(':'))
video.duration = datetime.timedelta(minutes=minutes, seconds=seconds)
else:
raise BrokenPageError('Unable to retrieve video duration')
video_file_urls = re.findall(r'"(http://media[^",]+\.flv)"', data)
if len(video_file_urls) == 0:
raise BrokenPageError('Video URL not found')
elif len(video_file_urls) > 1:
raise BrokenPageError('Many video file URL found')
else:
video.url = video_file_urls[0]
return video
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youjizz/test.py 0000664 0000000 0000000 00000002320 11666415431 0026750 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class YoujizzTest(BackendTest):
BACKEND = 'youjizz'
def test_youjizz(self):
self.assertTrue(len(self.backend.iter_search_results('anus', nsfw=False)) == 0)
l = list(self.backend.iter_search_results('sex', nsfw=True))
self.assertTrue(len(l) > 0)
v = l[0]
self.backend.fillobj(v, ('url',))
self.assertTrue(v.url and v.url.startswith('http://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youjizz/video.py 0000664 0000000 0000000 00000002106 11666415431 0027101 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.video import BaseVideo
__all__ = ['YoujizzVideo']
class YoujizzVideo(BaseVideo):
def __init__(self, *args, **kwargs):
BaseVideo.__init__(self, *args, **kwargs)
self.nsfw = True
self.ext = 'flv'
@classmethod
def id2url(cls, _id):
return 'http://www.youjizz.com/videos/%s.html' % _id
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youporn/ 0000775 0000000 0000000 00000000000 11666415431 0025412 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youporn/__init__.py 0000664 0000000 0000000 00000000102 11666415431 0027514 0 ustar 00root root 0000000 0000000 from .backend import YoupornBackend
__all__ = ['YoupornBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youporn/backend.py 0000664 0000000 0000000 00000004157 11666415431 0027362 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from weboob.capabilities.video import ICapVideo
from weboob.tools.backend import BaseBackend
from .browser import YoupornBrowser
from .video import YoupornVideo
__all__ = ['YoupornBackend']
class YoupornBackend(BaseBackend, ICapVideo):
NAME = 'youporn'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.9.1'
DESCRIPTION = 'Youporn videos website'
LICENSE = 'AGPLv3+'
BROWSER = YoupornBrowser
def get_video(self, _id):
with self.browser:
return self.browser.get_video(_id)
SORTBY = ['relevance', 'rating', 'views', 'time']
def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
if not nsfw:
return set()
with self.browser:
return self.browser.iter_search_results(pattern, self.SORTBY[sortby])
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
with self.browser:
video = self.browser.get_video(YoupornVideo.id2url(video.id), video)
if 'thumbnail' in fields and video.thumbnail:
with self.browser:
video.thumbnail.data = self.browser.readurl(video.thumbnail.url)
return video
OBJECTS = {YoupornVideo: fill_video}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youporn/browser.py 0000664 0000000 0000000 00000003354 11666415431 0027454 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser
from weboob.tools.browser.decorators import id2url
from .pages.index import IndexPage
from .pages.video import VideoPage
from .video import YoupornVideo
__all__ = ['YoupornBrowser']
class YoupornBrowser(BaseBrowser):
DOMAIN = 'youporn.com'
ENCODING = None
PAGES = {r'http://[w\.]*youporn\.com/?': IndexPage,
r'http://[w\.]*youporn\.com/search.*': IndexPage,
r'http://[w\.]*youporn\.com/watch/(?P\d+)/?.*': VideoPage,
r'http://[w\.]*youporngay\.com:80/watch/(?P.+)': VideoPage,
}
@id2url(YoupornVideo.id2url)
def get_video(self, url, video=None):
self.location(url)
return self.page.get_video(video)
def iter_search_results(self, pattern, sortby):
if not pattern:
self.home()
else:
self.location(self.buildurl('/search/%s' % sortby, query=pattern.encode('utf-8')))
assert self.is_on_page(IndexPage)
return self.page.iter_videos()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youporn/pages/ 0000775 0000000 0000000 00000000000 11666415431 0026511 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youporn/pages/__init__.py 0000664 0000000 0000000 00000000000 11666415431 0030610 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youporn/pages/base.py 0000664 0000000 0000000 00000002267 11666415431 0030004 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.mech import ClientForm
ControlNotFoundError = ClientForm.ControlNotFoundError
from mechanize import FormNotFoundError
from weboob.tools.browser import BasePage
__all__ = ['PornPage']
class PornPage(BasePage):
def on_loaded(self):
try:
self.browser.select_form(nr=0)
self.browser.submit(name='user_choice')
return False
except (ControlNotFoundError, FormNotFoundError):
return True
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youporn/pages/index.py 0000664 0000000 0000000 00000004737 11666415431 0030205 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import datetime
from .base import PornPage
from ..video import YoupornVideo
__all__ = ['IndexPage']
class IndexPage(PornPage):
def iter_videos(self):
uls = self.document.getroot().cssselect("ul[class=clearfix]")
if not uls:
return
for ul in uls:
for li in ul.findall('li'):
a = li.find('a')
if a is None or a.find('img') is None:
continue
thumbnail_url = a.find('img').attrib['src']
h1 = li.find('h1')
a = h1.find('a')
if a is None:
continue
url = a.attrib['href']
_id = url[len('/watch/'):]
_id = _id[:_id.find('/')]
title = a.text.strip()
minutes = seconds = 0
div = li.cssselect('div[class=duration_views]')
if div:
h2 = div[0].find('h2')
minutes = int(h2.text.strip())
seconds = int(h2.find('span').tail.strip())
rating = 0
rating_max = 0
div = li.cssselect('div[class=rating]')
if div:
p = div[0].find('p')
rating = float(p.text.strip())
rating_max = float(p.find('span').text.strip()[2:])
yield YoupornVideo(int(_id),
title=title,
rating=rating,
rating_max=rating_max,
duration=datetime.timedelta(minutes=minutes, seconds=seconds),
thumbnail_url=thumbnail_url,
)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youporn/pages/video.py 0000664 0000000 0000000 00000006571 11666415431 0030202 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
import datetime
from .base import PornPage
from ..video import YoupornVideo
class VideoPage(PornPage):
def get_video(self, video=None):
if not PornPage.on_loaded(self):
return
if video is None:
video = YoupornVideo(self.group_dict['id'])
video.title = self.get_title()
video.url, video.ext = self.get_url()
self.set_details(video)
return video
def get_url(self):
download_div = self.parser.select(self.document.getroot(), '#download', 1)
a = self.parser.select(download_div, 'a', 1)
m = re.match('^(\w+) - .*', a.text)
if m:
ext = m.group(1).lower()
else:
ext = 'flv'
return a.attrib['href'], ext
def get_title(self):
element = self.parser.select(self.document.getroot(), '#videoArea h1', 1)
return unicode(element.getchildren()[0].tail).strip()
DATE_REGEXP = re.compile("\w+ (\w+) (\d+) (\d+):(\d+):(\d+) (\d+)")
MONTH2I = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
def set_details(self, v):
details_div = self.parser.select(self.document.getroot(), '#details', 1)
for li in details_div.getiterator('li'):
span = li.find('span')
name = span.text.strip()
value = span.tail.strip()
if name == 'Duration:':
seconds = minutes = 0
for word in value.split():
if word.endswith('min'):
minutes = int(word[:word.find('min')])
elif word.endswith('sec'):
seconds = int(word[:word.find('sec')])
v.duration = datetime.timedelta(minutes=minutes, seconds=seconds)
elif name == 'Submitted:':
author = li.find('i')
if author is None:
author = li.find('a')
if author is None:
v.author = value
else:
v.author = author.text
elif name == 'Rating:':
r = value.split()
v.rating = float(r[0])
v.rating_max = float(r[2])
elif name == 'Date:':
m = self.DATE_REGEXP.match(value)
if m:
month = self.MONTH2I.index(m.group(1))
day = int(m.group(2))
hour = int(m.group(3))
minute = int(m.group(4))
second = int(m.group(5))
year = int(m.group(6))
v.date = datetime.datetime(year, month, day, hour, minute, second)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youporn/test.py 0000664 0000000 0000000 00000002406 11666415431 0026745 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class YoupornTest(BackendTest):
BACKEND = 'youporn'
def test_youporn(self):
self.assertTrue(len(self.backend.iter_search_results('penis', nsfw=False)) == 0)
l = list(self.backend.iter_search_results('ass to mouth', nsfw=True))
self.assertTrue(len(l) > 0)
v = l[0]
self.backend.fillobj(v, ('url',))
self.assertTrue(v.url and v.url.startswith('http://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
self.backend.browser.openurl(v.url)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youporn/video.py 0000664 0000000 0000000 00000002211 11666415431 0027066 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Roger Philibert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.video import BaseVideo
__all__ = ['YoupornVideo']
class YoupornVideo(BaseVideo):
def __init__(self, *args, **kwargs):
BaseVideo.__init__(self, *args, **kwargs)
self.nsfw = True
self.ext = 'flv'
@classmethod
def id2url(cls, _id):
if _id.isdigit():
return 'http://www.youporn.com/watch/%d' % int(_id)
else:
return None
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youtube/ 0000775 0000000 0000000 00000000000 11666415431 0025373 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youtube/__init__.py 0000664 0000000 0000000 00000001443 11666415431 0027506 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .backend import YoutubeBackend
__all__ = ['YoutubeBackend']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youtube/backend.py 0000664 0000000 0000000 00000012370 11666415431 0027337 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz, Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
import datetime
import gdata.youtube.service
import re
import urllib
from weboob.capabilities.video import ICapVideo
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.misc import to_unicode
from weboob.tools.value import ValueBackendPassword, Value
from .browser import YoutubeBrowser
from .video import YoutubeVideo
__all__ = ['YoutubeBackend']
class YoutubeBackend(BaseBackend, ICapVideo):
NAME = 'youtube'
MAINTAINER = 'Christophe Benz'
EMAIL = 'christophe.benz@gmail.com'
VERSION = '0.9.1'
DESCRIPTION = 'Youtube videos website'
LICENSE = 'AGPLv3+'
BROWSER = YoutubeBrowser
CONFIG = BackendConfig(Value('username', label='Email address', default=''),
ValueBackendPassword('password', label='Password', default=''))
URL_RE = re.compile(r'^https?://(?:\w*\.?youtube\.com/(?:watch\?v=|v/)|youtu\.be\/|\w*\.?youtube\.com\/user\/\w+#p\/u\/\d+\/)([^\?&]+)')
def create_default_browser(self):
password = None
username = self.config['username'].get()
if len(username) > 0:
password = self.config['password'].get()
return self.create_browser(username, password)
def _entry2video(self, entry):
"""
Parse an entry returned by gdata and return a Video object.
"""
video = YoutubeVideo(to_unicode(entry.id.text.split('/')[-1].strip()),
title=to_unicode(entry.media.title.text.strip()),
duration=to_unicode(datetime.timedelta(seconds=int(entry.media.duration.seconds.strip()))),
thumbnail_url=to_unicode(entry.media.thumbnail[0].url.strip()),
)
video.author = entry.author[0].name.text.strip()
if entry.media.name:
video.author = to_unicode(entry.media.name.text.strip())
return video
def _set_video_url(self, video):
"""
In the case of a download, if the user-chosen format is not
available, the next available format will be used.
Much of the code for this method is borrowed from youtubeservice.py of Cutetube
http://maemo.org/packages/view/cutetube/.
"""
if video.url:
return
player_url = YoutubeVideo.id2url(video.id)
with self.browser:
url, ext = self.browser.get_video_url(player_url)
video.url = url
video.ext = ext
def get_video(self, _id):
m = self.URL_RE.match(_id)
if m:
_id = m.group(1)
yt_service = gdata.youtube.service.YouTubeService()
try:
entry = yt_service.GetYouTubeVideoEntry(video_id=_id)
except gdata.service.Error, e:
if e.args[0]['status'] == 400:
return None
raise
video = self._entry2video(entry)
self._set_video_url(video)
return video
def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
YOUTUBE_MAX_RESULTS = 50
YOUTUBE_MAX_START_INDEX = 1000
yt_service = gdata.youtube.service.YouTubeService()
start_index = 1
nb_yielded = 0
while True:
query = gdata.youtube.service.YouTubeVideoQuery()
if pattern is not None:
if isinstance(pattern, unicode):
pattern = pattern.encode('utf-8')
query.vq = pattern
query.orderby = ('relevance', 'rating', 'viewCount', 'published')[sortby]
query.racy = 'include' if nsfw else 'exclude'
if max_results is None or max_results > YOUTUBE_MAX_RESULTS:
query_max_results = YOUTUBE_MAX_RESULTS
else:
query_max_results = max_results
query.max_results = query_max_results
if start_index > YOUTUBE_MAX_START_INDEX:
return
query.start_index = start_index
start_index += query_max_results
feed = yt_service.YouTubeQuery(query)
for entry in feed.entry:
yield self._entry2video(entry)
nb_yielded += 1
if nb_yielded == max_results:
return
def fill_video(self, video, fields):
if 'thumbnail' in fields:
video.thumbnail.data = urllib.urlopen(video.thumbnail.url).read()
if 'url' in fields:
self._set_video_url(video)
return video
OBJECTS = {YoutubeVideo: fill_video}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youtube/browser.py 0000664 0000000 0000000 00000004367 11666415431 0027442 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz, Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser import BaseBrowser
from .pages import BaseYoutubePage, VideoPage, ForbiddenVideoPage, \
VerifyAgePage, VerifyControversyPage, \
LoginPage, LoginRedirectPage
__all__ = ['YoutubeBrowser']
class YoutubeBrowser(BaseBrowser):
DOMAIN = u'youtube.com'
ENCODING = None
PAGES = {r'https?://.*youtube\.com/': BaseYoutubePage,
r'https?://.*youtube\.com/watch\?v=(?P.+)': VideoPage,
r'https?://.*youtube\.com/index\?ytsession=.+': ForbiddenVideoPage,
r'https?://.*youtube\.com/verify_age\?next_url=(?P.+)': VerifyAgePage,
r'https?://.*youtube\.com/verify_controversy\?next_url(?P.+)': VerifyControversyPage,
r'https?://accounts\.youtube\.com/accounts/SetSID.*': LoginRedirectPage,
r'https?://www.google.com/accounts/ServiceLogin.*': LoginPage,
}
def is_logged(self):
logged = not self.is_on_page(BaseYoutubePage) or self.page.is_logged()
return logged
def login(self):
self.location('https://www.google.com/accounts/ServiceLogin?uilel=3&service=youtube&passive=true&continue=http%3A%2F%2Fwww.youtube.com%2Fsignin%3Faction_handle_signin%3Dtrue%26nomobiletemp%3D1%26hl%3Den_US%26next%3D%252F&hl=en_US<mpl=sso')
self.page.login(self.username, self.password)
def get_video_url(self, player_url):
self.location(player_url)
assert self.is_on_page(VideoPage)
return self.page.get_video_url()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youtube/pages.py 0000664 0000000 0000000 00000010255 11666415431 0027047 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz, Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
try:
import json
except ImportError:
import simplejson as json
import urllib
from weboob.tools.browser import BasePage, BrokenPageError, BrowserIncorrectPassword
__all__ = ['LoginPage', 'LoginRedirectPage', 'ForbiddenVideo', 'ForbiddenVideoPage', \
'VerifyAgePage', 'VerifyControversyPage', 'VideoPage']
class LoginPage(BasePage):
def on_loaded(self):
errors = []
for errdiv in self.parser.select(self.document.getroot(), 'div.errormsg'):
errors.append(errdiv.text.encode('utf-8').strip())
if len(errors) > 0:
raise BrowserIncorrectPassword(', '.join(errors))
def login(self, username, password):
self.browser.select_form(predicate=lambda form: form.attrs.get('id', '') == 'gaia_loginform')
self.browser['Email'] = username
self.browser['Passwd'] = password
self.browser.submit()
class LoginRedirectPage(BasePage):
pass
class ForbiddenVideo(Exception):
pass
class BaseYoutubePage(BasePage):
def is_logged(self):
try:
self.parser.select(self.document.getroot(), 'span#masthead-user-expander', 1)
except BrokenPageError:
return False
else:
return True
class ForbiddenVideoPage(BaseYoutubePage):
def on_loaded(self):
element = self.parser.select(self.document.getroot(), '.yt-alert-content', 1)
raise ForbiddenVideo(element.text.strip())
class VerifyAgePage(BaseYoutubePage):
def on_loaded(self):
if not self.is_logged():
raise ForbiddenVideo('This video or group may contain content that is inappropriate for some users')
self.browser.select_form(predicate=lambda form: form.attrs.get('id', '') == 'confirm-age-form')
self.browser.submit()
class VerifyControversyPage(BaseYoutubePage):
def on_loaded(self):
self.browser.select_form(predicate=lambda form: 'verify_controversy' in form.attrs.get('action', ''))
self.browser.submit()
class VideoPage(BaseYoutubePage):
AVAILABLE_FORMATS = [38, 37, 45, 22, 43, 35, 34, 18, 6, 5, 17, 13]
FORMAT_EXTENSIONS = {
13: '3gp',
17: 'mp4',
18: 'mp4',
22: 'mp4',
37: 'mp4',
38: 'video', # You actually don't know if this will be MOV, AVI or whatever
43: 'webm',
45: 'webm',
}
def get_video_url(self, format=38):
formats = {}
for script in self.parser.select(self.document.getroot(), 'script'):
text = script.text
if not text:
continue
pattern = "'PLAYER_CONFIG': "
pos = text.find(pattern)
if pos < 0:
continue
sub = text[pos+len(pattern):pos+text[pos:].find('\n')]
a = json.loads(sub)
for part in a['args']['url_encoded_fmt_stream_map'].split('&'):
key, value = part.split('=', 1)
if key != 'itag' or not 'url' in value:
continue
value = urllib.unquote(value)
fmt, url = value.split(',url=')
formats[int(fmt)] = url
# choose the better format to use.
for format in self.AVAILABLE_FORMATS[self.AVAILABLE_FORMATS.index(format):]:
if format in formats:
url = formats.get(format)
ext = self.FORMAT_EXTENSIONS.get(format, 'flv')
return url, ext
raise BrokenPageError('Unable to find file URL')
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youtube/test.py 0000664 0000000 0000000 00000002312 11666415431 0026722 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
class YoutubeTest(BackendTest):
BACKEND = 'youtube'
def test_youtube(self):
l = list(self.backend.iter_search_results('lol'))
self.assertTrue(len(l) > 0)
v = l[0]
self.backend.fillobj(v, ('url',))
self.assertTrue(v.url and v.url.startswith('http://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
assert self.backend.get_video(v.shorturl)
self.backend.browser.openurl(v.url)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/backends/youtube/video.py 0000664 0000000 0000000 00000002050 11666415431 0027050 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.video import BaseVideo
__all__ = ['YoutubeVideo']
class YoutubeVideo(BaseVideo):
@classmethod
def id2url(cls, _id):
return 'http://www.youtube.com/watch?v=%s' % _id
def _get_shorturl(self):
return 'http://youtu.be/%s' % self.id
shorturl = property(_get_shorturl)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/ 0000775 0000000 0000000 00000000000 11666415431 0024556 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/__init__.py 0000664 0000000 0000000 00000000001 11666415431 0026656 0 ustar 00root root 0000000 0000000
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/account.py 0000664 0000000 0000000 00000004726 11666415431 0026575 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .base import IBaseCap, CapBaseObject
__all__ = ['ICapAccount']
class AccountRegisterError(Exception):
pass
class Account(CapBaseObject):
def __init__(self, id=None):
CapBaseObject.__init__(self, id)
self.add_field('login', basestring)
self.add_field('password', basestring)
self.add_field('properties', dict)
class StatusField(object):
FIELD_TEXT = 0x001 # the value is a long text
FIELD_HTML = 0x002 # the value is HTML formated
def __init__(self, key, label, value, flags=0):
self.key = key
self.label = label
self.value = value
self.flags = flags
class ICapAccount(IBaseCap):
# This class constant may be a list of Value* objects. If the value remains
# None, weboob considers that register_account() isn't supported.
ACCOUNT_REGISTER_PROPERTIES = None
@staticmethod
def register_account(account):
"""
Register an account on website
This is a static method, it would be called even if the backend is
instancied.
@param account an Account object which describe the account to create
"""
raise NotImplementedError()
def confirm_account(self, mail):
"""
From an email go to the confirm link.
"""
raise NotImplementedError()
def get_account(self):
"""
Get the current account.
"""
raise NotImplementedError()
def update_account(self, account):
"""
Update the current account.
"""
raise NotImplementedError()
def get_account_status(self):
"""
Get status of the current account.
@return a list of fields
"""
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/bank.py 0000664 0000000 0000000 00000006712 11666415431 0026051 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import datetime, date
from .base import CapBaseObject
from .collection import ICapCollection, CollectionNotFound
__all__ = ['Account', 'AccountNotFound', 'TransferError', 'ICapBank',
'Operation']
class AccountNotFound(Exception):
def __init__(self, msg=None):
if msg is None:
msg = 'Account not found'
Exception.__init__(self, msg)
class TransferError(Exception):
pass
class Recipient(CapBaseObject):
def __init__(self):
CapBaseObject.__init__(self, 0)
self.add_field('label', basestring)
class Account(Recipient):
def __init__(self):
Recipient.__init__(self)
self.add_field('balance', float)
self.add_field('coming', float)
self.link_id = None
def __repr__(self):
return u"" % (self.id, self.label)
class Operation(CapBaseObject):
def __init__(self, id):
CapBaseObject.__init__(self, id)
self.add_field('date', (basestring, datetime, date))
self.add_field('category', unicode)
self.add_field('label', unicode)
self.add_field('amount', float)
def __repr__(self):
return "" % (self.date,
self.label, self.amount)
class Transfer(CapBaseObject):
def __init__(self, id):
CapBaseObject.__init__(self, id)
self.add_field('amount', float)
self.add_field('date', (basestring, datetime, date))
self.add_field('origin', (int, long, basestring))
self.add_field('recipient', (int, long, basestring))
class ICapBank(ICapCollection):
def iter_resources(self, splited_path):
if len(splited_path) > 0:
raise CollectionNotFound()
return self.iter_accounts()
def iter_accounts(self):
raise NotImplementedError()
def get_account(self, _id):
raise NotImplementedError()
def iter_operations(self, account):
raise NotImplementedError()
def iter_history(self, account):
raise NotImplementedError()
def iter_transfer_recipients(self, account):
"""
Iter recipients availables for a transfer from a specific account.
@param account [Account] account which initiate the transfer
@return [iter(Recipient)]
"""
raise NotImplementedError()
def transfer(self, account, recipient, amount, reason=None):
"""
Make a transfer from an account to a recipient.
@param account [Account] account to take money
@param recipient [Recipient] account to send money
@param amount [float] amount
@param reason [str] reason of transfer
@return [Transfer] a Transfer object
"""
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/base.py 0000664 0000000 0000000 00000010666 11666415431 0026053 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz, Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.misc import iter_fields
__all__ = ['FieldNotFound', 'IBaseCap', 'NotAvailable', 'NotLoaded',
'CapBaseObject']
class FieldNotFound(Exception):
def __init__(self, obj, field):
Exception.__init__(self,
u'Field "%s" not found for object %s' % (field, obj))
class NotAvailableMeta(type):
def __str__(self):
return unicode(self).decode('utf-8')
def __unicode__(self):
return u'Not available'
def __nonzero__(self):
return False
class NotAvailable(object):
__metaclass__ = NotAvailableMeta
class NotLoadedMeta(type):
def __str__(self):
return unicode(self).decode('utf-8')
def __unicode__(self):
return u'Not loaded'
def __nonzero__(self):
return False
class NotLoaded(object):
__metaclass__ = NotLoadedMeta
class IBaseCap(object):
pass
class CapBaseObject(object):
FIELDS = None
_attribs = None
def __init__(self, id, backend=None):
self.id = id
self.backend = backend
def add_field(self, name, type, value=NotLoaded):
"""
Add a field in list, which needs to be of type @type.
@param name [str] name of field
@param type [class] type accepted (can be a tuple of types)
@param value [object] value set to attribute (default is NotLoaded)
"""
if not isinstance(self.FIELDS, list):
self.FIELDS = []
self.FIELDS.append(name)
if self._attribs is None:
self._attribs = {}
self._attribs[name] = self._AttribValue(type, value)
def __iscomplete__(self):
"""
Return True if the object is completed.
It is usefull when the object is a field of an other object which is
going to be filled.
The default behavior is to iter on fields (with iter_fields) and if
a field is NotLoaded, return False.
"""
for key, value in self.iter_fields():
if value is NotLoaded:
return False
return True
def iter_fields(self):
"""
Iterate on the FIELDS keys and values.
Can be overloaded to iterate on other things.
@return [iter(key,value)] iterator on key, value
"""
if self.FIELDS is None:
yield 'id', self.id
for key, value in iter_fields(self):
if key not in ('id', 'backend','FIELDS'):
yield key, value
else:
yield 'id', self.id
for attrstr in self.FIELDS:
yield attrstr, getattr(self, attrstr)
def __eq__(self, obj):
if isinstance(obj, CapBaseObject):
return self.backend == obj.backend and self.id == obj.id
else:
return False
class _AttribValue(object):
def __init__(self, type, value):
self.type = type
self.value = value
def __getattr__(self, name):
if self._attribs is not None and name in self._attribs:
return self._attribs[name].value
else:
raise AttributeError, "'%s' object has no attribute '%s'" % (
self.__class__.__name__, name)
def __setattr__(self, name, value):
try:
attr = (self._attribs or {})[name]
except KeyError:
object.__setattr__(self, name, value)
else:
if not isinstance(value, attr.type) and \
value is not NotLoaded and \
value is not NotAvailable and \
value is not None:
raise ValueError(
'Value for "%s" needs to be of type %r, not %r' % (
name, attr.type, type(value)))
attr.value = value
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/bugtracker.py 0000664 0000000 0000000 00000013627 11666415431 0027272 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import datetime, timedelta
from .base import IBaseCap, CapBaseObject
__all__ = ['ICapBugTracker']
class IssueError(Exception):
pass
class Project(CapBaseObject):
def __init__(self, id, name):
CapBaseObject.__init__(self, id)
self.add_field('name', unicode, name)
self.add_field('members', list)
self.add_field('versions', list)
self.add_field('categories', list)
self.add_field('statuses', list)
def __repr__(self):
return '' % self.name
def find_user(self, id, name):
for user in self.members:
if user.id == id:
return user
if name is None:
return None
return User(id, name)
def find_version(self, id, name):
for version in self.versions:
if version.id == id:
return version
if name is None:
return None
return Version(id, name)
def find_status(self, name):
for status in self.statuses:
if status.name == name:
return status
if name is None:
return None
return None
class User(CapBaseObject):
def __init__(self, id, name):
CapBaseObject.__init__(self, id)
self.add_field('name', unicode, name)
def __repr__(self):
return '' % self.name
class Version(CapBaseObject):
def __init__(self, id, name):
CapBaseObject.__init__(self, id)
self.add_field('name', unicode, name)
def __repr__(self):
return '' % self.name
class Status(CapBaseObject):
(VALUE_NEW,
VALUE_PROGRESS,
VALUE_RESOLVED,
VALUE_REJECTED) = range(4)
def __init__(self, id, name, value):
CapBaseObject.__init__(self, id)
self.add_field('name', unicode, name)
self.add_field('value', int, value)
def __repr__(self):
return '' % self.name
class Attachment(CapBaseObject):
def __init__(self, id):
CapBaseObject.__init__(self, id)
self.add_field('filename', basestring)
self.add_field('url', basestring)
def __repr__(self):
return '' % self.filename
class Change(CapBaseObject):
def __init__(self, id):
CapBaseObject.__init__(self, id)
self.add_field('field', unicode)
self.add_field('last', unicode)
self.add_field('new', unicode)
class Update(CapBaseObject):
def __init__(self, id):
CapBaseObject.__init__(self, id)
self.add_field('author', User)
self.add_field('date', datetime)
self.add_field('hours', timedelta)
self.add_field('message', unicode)
self.add_field('attachments', (list,tuple)) # Attachment
self.add_field('changes', (list,tuple)) # Change
def __repr__(self):
return '' % self.id
class Issue(CapBaseObject):
def __init__(self, id):
CapBaseObject.__init__(self, id)
self.add_field('project', Project)
self.add_field('title', unicode)
self.add_field('body', unicode)
self.add_field('creation', datetime)
self.add_field('updated', datetime)
self.add_field('attachments', (list,tuple))
self.add_field('history', (list,tuple))
self.add_field('author', User)
self.add_field('assignee', User)
self.add_field('category', unicode)
self.add_field('version', Version)
self.add_field('status', Status)
class Query(CapBaseObject):
def __init__(self):
CapBaseObject.__init__(self, '')
self.add_field('project', unicode)
self.add_field('title', unicode)
self.add_field('author', unicode)
self.add_field('assignee', unicode)
self.add_field('version', unicode)
self.add_field('category', unicode)
self.add_field('status', unicode)
class ICapBugTracker(IBaseCap):
def iter_issues(self, query):
"""
Iter issues with optionnal patterns.
@param query [Query]
@return [iter(Issue)] issues
"""
raise NotImplementedError()
def get_issue(self, id):
"""
Get an issue from its ID.
@return Issue
"""
raise NotImplementedError()
def create_issue(self, project):
"""
Create an empty issue on the given project.
@return [Issue] the created issue.
"""
raise NotImplementedError()
def post_issue(self, issue):
"""
Post an issue to create or update it.
"""
raise NotImplementedError()
def update_issue(self, issue, update):
"""
Add an update to an issue.
@param issue [id,Issue] issue or id of issue
@param update [Update] an Update object
"""
raise NotImplementedError()
def remove_issue(self, issue):
"""
Remove an issue.
"""
raise NotImplementedError()
def iter_projects(self):
"""
Iter projects.
@return [iter(Project)] projects
"""
raise NotImplementedError()
def get_project(self, id):
"""
Get a project from its ID.
@return [Project]
"""
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/chat.py 0000664 0000000 0000000 00000002776 11666415431 0026063 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import datetime
from .base import IBaseCap, CapBaseObject
__all__ = ['ChatException', 'ICapChat']
class ChatException(Exception):
pass
class ChatMessage(CapBaseObject):
def __init__(self, id_from, id_to, message, date=None):
CapBaseObject.__init__(self, '%s.%s' % (id_from, id_to))
self.add_field('id_from', basestring, id_from)
self.add_field('id_to', basestring, id_to)
self.add_field('message', basestring, message)
self.add_field('date', datetime.datetime, date)
if self.date is None:
self.date = datetime.datetime.utcnow()
class ICapChat(IBaseCap):
def iter_chat_messages(self, _id=None):
raise NotImplementedError()
def send_chat_message(self, _id, message):
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/collection.py 0000664 0000000 0000000 00000003411 11666415431 0027262 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .base import IBaseCap
__all__ = ['ICapCollection', 'Collection', 'CollectionNotFound']
class CollectionNotFound(Exception):
def __init__(self, msg=None):
if msg is None:
msg = 'Collection not found'
Exception.__init__(self, msg)
class Children(object):
def __get__(self, obj, type=None):
if callable(obj._childrenfct):
return obj._childrenfct(obj.id)
else:
return obj._children
def __set__(self, obj, value):
obj._childrenfct = value
class Collection(object):
"""
_childrenfct
_children
appendchild
children return iterator
"""
children = Children()
def __init__(self, title=None, children=None):
self.title = title
self._children = children if children else []
self._childrenfct = None
def appendchild(self, child):
self._children.append(child)
class Ressource(object):
pass
class ICapCollection(IBaseCap):
def iter_resources(self, splited_path):
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/contact.py 0000664 0000000 0000000 00000007760 11666415431 0026575 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .base import IBaseCap, CapBaseObject
from weboob.tools.ordereddict import OrderedDict
__all__ = ['ICapContact', 'Contact']
class ProfileNode(object):
HEAD = 0x01
SECTION = 0x02
def __init__(self, name, label, value, sufix=None, flags=0):
self.name = name
self.label = label
self.value = value
self.sufix = sufix
self.flags = flags
def __getitem__(self, key):
return self.value[key]
class ContactPhoto(CapBaseObject):
def __init__(self, name):
CapBaseObject.__init__(self, name)
self.add_field('name', basestring, name)
self.add_field('url', basestring)
self.add_field('data', str)
self.add_field('thumbnail_url', basestring)
self.add_field('thumbnail_data', basestring)
self.add_field('hidden', bool, False)
def __iscomplete__(self):
return (self.data and (not self.thumbnail_url or self.thumbnail_data))
def __str__(self):
return self.url
def __repr__(self):
return u'' % (self.id,
len(self.data) if self.data else 0,
len(self.thumbnail_data) if self.thumbnail_data else 0)
class Contact(CapBaseObject):
STATUS_ONLINE = 0x001
STATUS_AWAY = 0x002
STATUS_OFFLINE = 0x004
STATUS_ALL = 0xfff
def __init__(self, id, name, status):
CapBaseObject.__init__(self, id)
self.add_field('name', basestring, name)
self.add_field('status', int, status)
self.add_field('url', basestring)
self.add_field('status_msg', basestring)
self.add_field('summary', basestring)
self.add_field('photos', dict, OrderedDict())
self.add_field('profile', dict)
def set_photo(self, name, **kwargs):
if not name in self.photos:
self.photos[name] = ContactPhoto(name)
photo = self.photos[name]
for key, value in kwargs.iteritems():
setattr(photo, key, value)
class QueryError(Exception):
pass
class Query(CapBaseObject):
def __init__(self, id, message):
CapBaseObject.__init__(self, id)
self.add_field('message', basestring, message)
class ICapContact(IBaseCap):
def iter_contacts(self, status=Contact.STATUS_ALL, ids=None):
"""
Iter contacts
@param status get only contacts with the specified status
@param ids if set, get the specified contacts
@return iterator over the contacts found
"""
raise NotImplementedError()
def get_contact(self, id):
"""
Get a contact from his id.
The default implementation only calls iter_contacts()
with the proper values, but it might be overloaded
by backends.
@param id the ID requested
@return the Contact object, or None if not found
"""
l = self.iter_contacts(ids=[id])
try:
return l[0]
except IndexError:
return None
def send_query(self, id):
"""
Send a query to a contact
@param id the ID of contact
@return a Query object
@except QueryError
"""
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/content.py 0000664 0000000 0000000 00000003406 11666415431 0026605 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .base import IBaseCap, CapBaseObject
from datetime import datetime
class Content(CapBaseObject):
def __init__(self, id):
CapBaseObject.__init__(self, id)
self.add_field('title', basestring)
self.add_field('author', basestring)
self.add_field('content', basestring)
self.add_field('revision', basestring)
class Revision(CapBaseObject):
def __init__(self, _id):
CapBaseObject.__init__(self, _id)
self.add_field('author', basestring)
self.add_field('comment', basestring)
self.add_field('revision', basestring)
self.add_field('timestamp', datetime)
self.add_field('minor', bool)
class ICapContent(IBaseCap):
def get_content(self, id, revision=None):
raise NotImplementedError()
def iter_revisions(self, id, max_results=10):
raise NotImplementedError()
def push_content(self, content, message=None, minor=False):
raise NotImplementedError()
def get_content_preview(self, content):
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/dating.py 0000664 0000000 0000000 00000003631 11666415431 0026401 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .base import IBaseCap
__all__ = ['ICapDating']
class OptimizationNotFound(Exception):
pass
class Optimization(object):
# Configuration of optim can be made by Value*s in this dict.
CONFIG = {}
def start(self):
raise NotImplementedError()
def stop(self):
raise NotImplementedError()
def is_running(self):
raise NotImplementedError()
def get_config(self):
return None
def set_config(self, params):
raise NotImplementedError()
class ICapDating(IBaseCap):
def init_optimizations(self):
raise NotImplementedError()
def add_optimization(self, name, optim):
setattr(self, 'OPTIM_%s' % name, optim)
def iter_optimizations(self, *optims):
for attr_name in dir(self):
if not attr_name.startswith('OPTIM_'):
continue
attr = getattr(self, attr_name)
if attr is None:
continue
yield attr_name[6:], attr
def get_optimization(self, optim):
optim = optim.upper()
if not hasattr(self, 'OPTIM_%s' % optim):
raise OptimizationNotFound()
return getattr(self, 'OPTIM_%s' % optim)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/gallery.py 0000664 0000000 0000000 00000007316 11666415431 0026576 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Christophe Benz, Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import datetime
from weboob.tools.capabilities.thumbnail import Thumbnail
from .base import IBaseCap, CapBaseObject, NotLoaded
__all__ = ['Thumbnail', 'ICapGallery', 'BaseGallery', 'BaseImage']
class BaseGallery(CapBaseObject):
"""
Represents a gallery.
This object has to be inherited to specify how to calculate the URL of the gallery from its ID.
"""
def __init__(self, _id, title=NotLoaded, url=NotLoaded, cardinality=NotLoaded, date=NotLoaded,
rating=NotLoaded, rating_max=NotLoaded, thumbnail=NotLoaded, thumbnail_url=None, nsfw=False):
CapBaseObject.__init__(self, unicode(_id))
self.add_field('title', basestring, title)
self.add_field('url', basestring, url)
self.add_field('description', basestring)
self.add_field('cardinality', int)
self.add_field('date', datetime, date)
self.add_field('rating', (int,long,float), rating)
self.add_field('rating_max', (int,long,float), rating_max)
self.add_field('thumbnail', Thumbnail, thumbnail)
@classmethod
def id2url(cls, _id):
"""Overloaded in child classes provided by backends."""
raise NotImplementedError()
@property
def page_url(self):
return self.id2url(self.id)
def iter_image(self):
raise NotImplementedError()
class BaseImage(CapBaseObject):
def __init__(self, _id, index=None, thumbnail=NotLoaded, url=NotLoaded,
ext=NotLoaded, gallery=None):
CapBaseObject.__init__(self, unicode(_id))
self.add_field('index', int, index) # usually page number
self.add_field('thumbnail', Thumbnail, thumbnail)
self.add_field('url', basestring, url)
self.add_field('ext', basestring, ext)
self.add_field('data', str)
self.add_field('gallery', BaseGallery, gallery)
def __str__(self):
return self.url
def __repr__(self):
return '' % self.url
def __iscomplete__(self):
return self.data is not NotLoaded
class ICapGallery(IBaseCap):
"""
This capability represents the ability for a website backend to provide videos.
"""
(SEARCH_RELEVANCE,
SEARCH_RATING,
SEARCH_VIEWS,
SEARCH_DATE) = range(4)
def iter_search_results(self, pattern=None, sortby=SEARCH_RELEVANCE, max_results=None):
"""
Iter results of a search on a pattern. Note that if pattern is None,
it get the latest videos.
@param pattern [str] pattern to search on
@param sortby [enum] sort by...
@param nsfw [bool] include non-suitable for work videos if True
@param max_results [int] maximum number of results to return
"""
raise NotImplementedError()
def get_gallery(self, _id):
"""
Get gallery from an ID.
@param _id the gallery id. It can be a numeric ID, or a page url, or so.
@return a Gallery object
"""
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/geolocip.py 0000664 0000000 0000000 00000002645 11666415431 0026740 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .base import IBaseCap, CapBaseObject
__all__ = ['IpLocation', 'ICapGeolocIp']
class IpLocation(CapBaseObject):
def __init__(self, ipaddr):
CapBaseObject.__init__(self, ipaddr)
self.ipaddr = ipaddr
self.add_field('city', basestring)
self.add_field('region', basestring)
self.add_field('zipcode', basestring)
self.add_field('country', basestring)
self.add_field('lt', float)
self.add_field('lg', float)
self.add_field('host', basestring)
self.add_field('tld', basestring)
self.add_field('isp', basestring)
class ICapGeolocIp(IBaseCap):
def get_location(self, ipaddr):
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/messages.py 0000664 0000000 0000000 00000012256 11666415431 0026745 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import datetime
import time
from .base import IBaseCap, CapBaseObject, NotLoaded
__all__ = ['ICapMessages', 'ICapMessagesPost', 'Message', 'Thread', 'CantSendMessage']
class Message(CapBaseObject):
IS_HTML = 0x001 # The content is HTML formatted
IS_UNREAD = 0x002 # The message is unread
IS_ACCUSED = 0x004 # The receiver has read this message
IS_NOT_ACCUSED = 0x008 # The receiver has not read this message
def __init__(self, thread, id,
title=NotLoaded,
sender=NotLoaded,
receivers=NotLoaded,
date=None,
parent=NotLoaded,
content=NotLoaded,
signature=NotLoaded,
children=NotLoaded,
flags=0):
CapBaseObject.__init__(self, id)
assert thread is not None
self.add_field('thread', Thread, thread)
self.add_field('title', basestring, title)
self.add_field('sender', basestring, sender)
self.add_field('receivers', list, receivers)
self.add_field('date', (datetime.datetime, datetime.date), date)
self.add_field('parent', Message, parent)
self.add_field('content', basestring, content)
self.add_field('signature', basestring, signature)
self.add_field('children', list, children)
self.add_field('flags', int, flags)
if date is None:
date = datetime.datetime.utcnow()
self.date = date
if isinstance(parent, Message):
self.parent = parent
else:
self.parent = NotLoaded
self._parent_id = parent
@property
def date_int(self):
return int(time.strftime('%Y%m%d%H%M%S', self.date.timetuple()))
@property
def full_id(self):
return '%s.%s' % (self.thread.id, self.id)
@property
def full_parent_id(self):
if self.parent:
return self.parent.full_id
elif self._parent_id is None:
return ''
elif self._parent_id is NotLoaded:
return NotLoaded
else:
return '%s.%s' % (self.thread.id, self._parent_id)
def __eq__(self, msg):
if self.thread:
return unicode(self.thread.id) == unicode(msg.thread.id) and \
unicode(self.id) == unicode(msg.id)
else:
return unicode(self.id) == unicode(msg.id)
def __repr__(self):
return '' % (
self.full_id, self.title, self.date, self.sender)
class Thread(CapBaseObject):
IS_THREADS = 0x001
IS_DISCUSSION = 0x002
def __init__(self, id):
CapBaseObject.__init__(self, id)
self.add_field('root', Message)
self.add_field('title', basestring)
self.add_field('date', (datetime.datetime, datetime.date))
self.add_field('nb_messages', int)
self.add_field('nb_unread', int)
self.add_field('flags', int, self.IS_THREADS)
def iter_all_messages(self):
if self.root:
yield self.root
for m in self._iter_all_messages(self.root):
yield m
def _iter_all_messages(self, message):
if message.children:
for child in message.children:
yield child
for m in self._iter_all_messages(child):
yield m
class ICapMessages(IBaseCap):
def iter_threads(self):
"""
Iterates on threads, from newers to olders.
@return [iter] Thread objects
"""
raise NotImplementedError()
def get_thread(self, id):
"""
Get a specific thread.
@return [Thread] the Thread object
"""
raise NotImplementedError()
def iter_unread_messages(self, thread=None):
"""
Iterates on messages which hasn't been marked as read.
@param thread thread name (optional)
@return [iter] Message objects
"""
raise NotImplementedError()
def set_message_read(self, message):
"""
Set a message as read.
@param [message] message read (or ID)
"""
raise NotImplementedError()
class CantSendMessage(Exception):
pass
class ICapMessagesPost(IBaseCap):
def post_message(self, message):
"""
Post a message.
@param message Message object
@return
"""
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/paste.py 0000664 0000000 0000000 00000006162 11666415431 0026251 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .base import IBaseCap, CapBaseObject, NotLoaded
__all__ = ['PasteNotFound', 'BasePaste', 'ICapPaste']
class PasteNotFound(Exception):
pass
class BasePaste(CapBaseObject):
"""
Represents a pasted text.
"""
def __init__(self, _id, title=NotLoaded, language=NotLoaded, contents=NotLoaded,
public=NotLoaded):
CapBaseObject.__init__(self, unicode(_id))
self.add_field('title', basestring, title)
self.add_field('language', basestring, language)
self.add_field('contents', basestring, contents)
self.add_field('public', bool, public)
@classmethod
def id2url(cls, _id):
"""Overloaded in child classes provided by backends."""
raise NotImplementedError()
@property
def page_url(self):
return self.id2url(self.id)
class ICapPaste(IBaseCap):
"""
This capability represents the ability for a website backend to store plain text.
"""
def new_paste(self, *args, **kwargs):
"""
Get a new paste object for posting it with the backend.
The parameters should be passed to the object init.
@return a Paste object
"""
raise NotImplementedError()
def can_post(self, contents, title=None, public=None, max_age=None):
"""
Checks if the paste can be pasted by this backend.
Some properties are considered required (public/private, max_age) while others
are just bonuses (language).
contents: Can be used to check encodability, maximum length, etc.
title: Can be used to check length, allowed characters. Should not be required.
public: True must be public, False must be private, None do not care.
max_age: Maximum time to live in seconds.
A score of 0 means the backend is not suitable.
A score of 1 means the backend is suitable.
Higher scores means it is more suitable than others with a lower score.
@return int Score
"""
raise NotImplementedError()
def get_paste(self, url):
"""
Get a Paste from an ID or URL.
@param _id the paste id. It can be an ID or a page URL.
@return a Paste object
"""
raise NotImplementedError()
def post_paste(self, paste, max_age=None):
"""
Post a paste.
@param paste Paste object
@return
"""
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/radio.py 0000664 0000000 0000000 00000004002 11666415431 0026222 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .base import IBaseCap, CapBaseObject
__all__ = ['Emission', 'Stream', 'Radio', 'ICapRadio']
class Emission(CapBaseObject):
def __init__(self, id):
CapBaseObject.__init__(self, id)
self.add_field('artist', unicode)
self.add_field('title', unicode)
def __iscomplete__(self):
# This volatile information may be reloaded everytimes.
return False
def __unicode__(self):
if self.artist:
return u'%s - %s' % (self.artist, self.title)
else:
return self.title
class Stream(CapBaseObject):
def __init__(self, id):
CapBaseObject.__init__(self, id)
self.add_field('title', unicode)
self.add_field('url', unicode)
def __unicode__(self):
return u'%s (%s)' % (self.title, self.url)
def __repr__(self):
return self.__unicode__()
class Radio(CapBaseObject):
def __init__(self, id):
CapBaseObject.__init__(self, id)
self.add_field('title', unicode)
self.add_field('description', unicode)
self.add_field('current', Emission)
self.add_field('streams', list)
class ICapRadio(IBaseCap):
def iter_radios_search(self, pattern):
raise NotImplementedError()
def get_radio(self, id):
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/torrent.py 0000664 0000000 0000000 00000003602 11666415431 0026626 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import datetime
from .base import IBaseCap, CapBaseObject, NotLoaded
__all__ = ['ICapTorrent', 'Torrent']
class Torrent(CapBaseObject):
def __init__(self, id, name, date=NotLoaded, size=NotLoaded, url=NotLoaded,
seeders=NotLoaded, leechers=NotLoaded, files=NotLoaded,
description=NotLoaded, filename=NotLoaded):
CapBaseObject.__init__(self, id)
self.add_field('name', basestring, name)
self.add_field('size', (int,long,float), size)
self.add_field('date', datetime, date)
self.add_field('url', basestring, url)
self.add_field('seeders', int, seeders)
self.add_field('leechers', int, leechers)
self.add_field('files', list, files)
self.add_field('description', basestring, description)
self.add_field('filename', basestring, filename) # suggested name of the .torrent file
class ICapTorrent(IBaseCap):
def iter_torrents(self, pattern):
raise NotImplementedError()
def get_torrent(self, _id):
raise NotImplementedError()
def get_torrent_file(self, _id):
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/travel.py 0000664 0000000 0000000 00000006467 11666415431 0026442 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import time, datetime, timedelta
from .base import IBaseCap, CapBaseObject
__all__ = ['Departure', 'ICapTravel', 'Station']
class Station(CapBaseObject):
def __init__(self, id, name):
CapBaseObject.__init__(self, id)
self.add_field('name', basestring, name)
def __repr__(self):
return "" % (self.id, self.name)
class Departure(CapBaseObject):
def __init__(self, id, _type, _time):
CapBaseObject.__init__(self, id)
self.add_field('type', basestring, _type)
self.add_field('time', datetime, _time)
self.add_field('departure_station', basestring)
self.add_field('arrival_station', basestring)
self.add_field('late', time, time())
self.add_field('information', basestring)
self.add_field('plateform', basestring)
def __repr__(self):
return u"" % (
self.id, self.type, self.time.strftime('%H:%M'), self.departure_station, self.arrival_station)
class RoadStep(CapBaseObject):
def __init__(self, id):
CapBaseObject.__init__(self, id)
self.add_field('line', basestring)
self.add_field('start_time', time)
self.add_field('end_time', time)
self.add_field('departure', unicode)
self.add_field('arrival', unicode)
self.add_field('duration', timedelta)
class RoadmapError(Exception):
pass
class RoadmapFilters(CapBaseObject):
def __init__(self):
CapBaseObject.__init__(self, '')
self.add_field('departure_time', datetime)
self.add_field('arrival_time', datetime)
class ICapTravel(IBaseCap):
def iter_station_search(self, pattern):
"""
Iterates on search results of stations.
@param pattern [str] the search pattern
@return [iter] the of Station objects
"""
raise NotImplementedError()
def iter_station_departures(self, station_id, arrival_id):
"""
Iterate on departures.
@param station_id [id] the station id
@param arrival_id [id] optionnal arrival station id
@return [iter] result of Departure objects
"""
raise NotImplementedError()
def iter_roadmap(self, departure, arrival, filters):
"""
Get a roadmap.
@param departure [str] name of departure station
@param arrival [str] name of arrival station
@param filters [RoadmapFilters] filters on search
@return [iter(RoadStep)] steps of roadmap
"""
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/video.py 0000664 0000000 0000000 00000006407 11666415431 0026245 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import datetime, timedelta
from .base import IBaseCap, CapBaseObject, NotLoaded
from weboob.tools.capabilities.thumbnail import Thumbnail
__all__ = ['BaseVideo', 'ICapVideo']
class BaseVideo(CapBaseObject):
"""
Represents a video.
This object has to be inherited to specify how to calculate the URL of the video from its ID.
"""
def __init__(self, _id, title=NotLoaded, url=NotLoaded, author=NotLoaded, duration=NotLoaded, date=NotLoaded,
rating=NotLoaded, rating_max=NotLoaded, thumbnail=NotLoaded, thumbnail_url=None, nsfw=False):
CapBaseObject.__init__(self, unicode(_id))
self.add_field('title', basestring, title)
self.add_field('url', basestring, url)
self.add_field('ext', basestring)
self.add_field('author', basestring, author)
self.add_field('description', basestring)
self.add_field('duration', (int,long,timedelta), duration)
self.add_field('date', datetime, date)
self.add_field('rating', (int,long,float), rating)
self.add_field('rating_max', (int,long,float), rating_max)
self.add_field('thumbnail', Thumbnail, thumbnail)
self.add_field('nsfw', bool, nsfw)
# XXX remove this and fix all backends
if thumbnail_url is not None and self.thumbnail is NotLoaded:
self.thumbnail = Thumbnail(thumbnail_url)
@classmethod
def id2url(cls, _id):
"""Overloaded in child classes provided by backends."""
raise NotImplementedError()
@property
def page_url(self):
return self.id2url(self.id)
class ICapVideo(IBaseCap):
"""
This capability represents the ability for a website backend to provide videos.
"""
(SEARCH_RELEVANCE,
SEARCH_RATING,
SEARCH_VIEWS,
SEARCH_DATE) = range(4)
def iter_search_results(self, pattern=None, sortby=SEARCH_RELEVANCE, nsfw=False, max_results=None):
"""
Iter results of a search on a pattern. Note that if pattern is None,
it get the latest videos.
@param pattern [str] pattern to search on
@param sortby [enum] sort by...
@param nsfw [bool] include non-suitable for work videos if True
@param max_results [int] maximum number of results to return
"""
raise NotImplementedError()
def get_video(self, _id):
"""
Get a Video from an ID.
@param _id the video id. It can be a numeric ID, or a page url, or so.
@return a Video object
"""
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/capabilities/weather.py 0000664 0000000 0000000 00000003777 11666415431 0026605 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import datetime
from .base import IBaseCap, CapBaseObject
__all__ = ['City', 'CityNotFound', 'Current', 'Forecast', 'ICapWeather']
class Forecast(CapBaseObject):
def __init__(self, date, low, high, text, unit):
CapBaseObject.__init__(self, date)
self.add_field('date', (basestring,datetime), date)
self.add_field('low', (int,float), low)
self.add_field('high', (int,float), high)
self.add_field('text', basestring, text)
self.add_field('unit', basestring, unit)
class Current(CapBaseObject):
def __init__(self, date, temp, text, unit):
CapBaseObject.__init__(self, date)
self.add_field('date', (basestring,datetime), date)
self.add_field('text', basestring, text)
self.add_field('temp', (int,float), temp)
self.add_field('unit', basestring, unit)
class City(CapBaseObject):
def __init__(self, id, name):
CapBaseObject.__init__(self, id)
self.add_field('name', basestring, name)
class CityNotFound(Exception):
pass
class ICapWeather(IBaseCap):
def iter_city_search(self, pattern):
raise NotImplementedError()
def get_current(self, city_id):
raise NotImplementedError()
def iter_forecast(self, city_id):
raise NotImplementedError()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/core/ 0000775 0000000 0000000 00000000000 11666415431 0023055 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/core/__init__.py 0000664 0000000 0000000 00000001501 11666415431 0025163 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .bcall import CallErrors
from .ouiboube import Weboob
__all__ = ['CallErrors', 'Weboob']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/core/backendscfg.py 0000664 0000000 0000000 00000011357 11666415431 0025670 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
import stat
import os
import sys
from ConfigParser import RawConfigParser, DuplicateSectionError
from logging import warning
__all__ = ['BackendsConfig', 'BackendAlreadyExists']
class BackendAlreadyExists(Exception):
pass
class BackendsConfig(object):
class WrongPermissions(Exception):
pass
def __init__(self, confpath):
self.confpath = confpath
try:
mode = os.stat(confpath).st_mode
except OSError:
if sys.platform == 'win32':
fptr = open(confpath,'w')
fptr.close()
else:
try:
os.mknod(confpath, 0600)
except OSError:
fptr = open(confpath,'w')
fptr.close()
os.chmod(confpath, 0600)
else:
if sys.platform != 'win32':
if mode & stat.S_IRGRP or mode & stat.S_IROTH:
raise self.WrongPermissions(
u'Weboob will not start until config file %s is readable by group or other users.' % confpath)
def iter_backends(self):
config = RawConfigParser()
config.read(self.confpath)
for instance_name in config.sections():
params = dict(config.items(instance_name))
try:
backend_name = params.pop('_backend')
except KeyError:
try:
backend_name = params.pop('_type')
warning(u'Please replace _type with _backend in your config file "%s", for backend "%s"' % (
self.confpath, backend_name))
except KeyError:
warning('Missing field "_backend" for configured backend "%s"', instance_name)
continue
yield instance_name, backend_name, params
def backend_exists(self, name):
"""
Return True if the backend exists in config.
"""
config = RawConfigParser()
config.read(self.confpath)
return name in config.sections()
def add_backend(self, instance_name, backend_name, params, edit=False):
if not instance_name:
raise ValueError(u'Please give a name to the configured backend.')
config = RawConfigParser()
config.read(self.confpath)
if not edit:
try:
config.add_section(instance_name)
except DuplicateSectionError:
raise BackendAlreadyExists(instance_name)
config.set(instance_name, '_backend', backend_name)
for key, value in params.iteritems():
if isinstance(value, unicode):
value = value.encode('utf-8')
config.set(instance_name, key, value)
with open(self.confpath, 'wb') as f:
config.write(f)
def edit_backend(self, instance_name, backend_name, params):
return self.add_backend(instance_name, backend_name, params, True)
def get_backend(self, instance_name):
config = RawConfigParser()
config.read(self.confpath)
if not config.has_section(instance_name):
raise KeyError(u'Configured backend "%s" not found' % instance_name)
items = dict(config.items(instance_name))
try:
backend_name = items.pop('_backend')
except KeyError:
try:
backend_name = items.pop('_type')
warning(u'Please replace _type with _backend in your config file "%s"' % self.confpath)
except KeyError:
warning('Missing field "_backend" for configured backend "%s"', instance_name)
raise KeyError(u'Configured backend "%s" not found' % instance_name)
return backend_name, items
def remove_backend(self, instance_name):
config = RawConfigParser()
config.read(self.confpath)
if not config.remove_section(instance_name):
return False
with open(self.confpath, 'w') as f:
config.write(f)
return True
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/core/bcall.py 0000664 0000000 0000000 00000016137 11666415431 0024514 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from copy import copy
from threading import Thread, Event, RLock, Timer
from weboob.capabilities.base import CapBaseObject
from weboob.tools.misc import get_backtrace
from weboob.tools.log import getLogger
__all__ = ['BackendsCall', 'CallErrors', 'IResultsCondition', 'ResultsConditionError']
class CallErrors(Exception):
def __init__(self, errors):
Exception.__init__(self, 'Errors during backend calls')
self.errors = copy(errors)
def __iter__(self):
return self.errors.__iter__()
class IResultsCondition(object):
def is_valid(self, obj):
raise NotImplementedError()
class ResultsConditionError(Exception):
pass
class BackendsCall(object):
def __init__(self, backends, condition, function, *args, **kwargs):
"""
@param backends list of backends to call.
@param condition a IResultsCondition object. Can be None.
@param function backends' method name, or callable object.
@param args, kwargs arguments given to called functions.
"""
self.logger = getLogger('bcall')
# Store if a backend is finished
self.backends = {}
for backend in backends:
self.backends[backend.name] = False
# Condition
self.condition = condition
# Global mutex on object
self.mutex = RLock()
# Event set when every backends have give their data
self.finish_event = Event()
# Event set when there are new responses
self.response_event = Event()
# Waiting responses
self.responses = []
# Errors
self.errors = []
# Threads
self.threads = []
# Create jobs for each backend
with self.mutex:
for backend in backends:
self.logger.debug('Creating a new thread for %s' % backend)
self.threads.append(Timer(0, self._caller, (backend, function, args, kwargs)).start())
if not backends:
self.finish_event.set()
def _store_error(self, backend, error):
with self.mutex:
backtrace = get_backtrace(error)
self.errors.append((backend, error, backtrace))
def _store_result(self, backend, result):
with self.mutex:
if isinstance(result, CapBaseObject):
if self.condition and not self.condition.is_valid(result):
return
result.backend = backend.name
self.responses.append((backend, result))
self.response_event.set()
def _caller(self, backend, function, args, kwargs):
self.logger.debug('%s: Thread created successfully' % backend)
with backend:
try:
# Call method on backend
try:
self.logger.debug('%s: Calling function %s' % (backend, function))
if callable(function):
result = function(backend, *args, **kwargs)
else:
result = getattr(backend, function)(*args, **kwargs)
except Exception, error:
self.logger.debug('%s: Called function %s raised an error: %r' % (backend, function, error))
self._store_error(backend, error)
else:
self.logger.debug('%s: Called function %s returned: %r' % (backend, function, result))
if hasattr(result, '__iter__') and not isinstance(result, basestring):
# Loop on iterator
try:
for subresult in result:
# Lock mutex only in loop in case the iterator is slow
# (for example if backend do some parsing operations)
self._store_result(backend, subresult)
except Exception, error:
self._store_error(backend, error)
else:
self._store_result(backend, result)
finally:
with self.mutex:
# This backend is now finished
self.backends[backend.name] = True
for finished in self.backends.itervalues():
if not finished:
return
self.response_event.set()
self.finish_event.set()
def _callback_thread_run(self, callback, errback):
responses = []
while not self.finish_event.isSet() or self.response_event.isSet():
self.response_event.wait()
with self.mutex:
responses = self.responses
self.responses = []
# Reset event
self.response_event.clear()
# Consume responses
while responses:
callback(*responses.pop(0))
if errback:
with self.mutex:
while self.errors:
errback(*self.errors.pop(0))
callback(None, None)
def callback_thread(self, callback, errback=None):
"""
Call this method to create a thread which will callback a
specified function everytimes a new result comes.
When the process is over, the function will be called with
both arguments set to None.
The functions prototypes:
def callback(backend, result)
def errback(backend, error)
"""
thread = Thread(target=self._callback_thread_run, args=(callback, errback))
thread.start()
return thread
def wait(self):
self.finish_event.wait()
with self.mutex:
if self.errors:
raise CallErrors(self.errors)
def __iter__(self):
# Don't know how to factorize with _callback_thread_run
responses = []
while not self.finish_event.isSet() or self.response_event.isSet():
self.response_event.wait()
with self.mutex:
responses = self.responses
self.responses = []
# Reset event
self.response_event.clear()
# Consume responses
while responses:
yield responses.pop(0)
# Raise errors
with self.mutex:
if self.errors:
raise CallErrors(self.errors)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/core/modules.py 0000664 0000000 0000000 00000011202 11666415431 0025073 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import logging
import os
from weboob.tools.backend import BaseBackend
from weboob.tools.log import getLogger
__all__ = ['Module', 'ModulesLoader', 'ModuleLoadError']
class ModuleLoadError(Exception):
def __init__(self, module_name, msg):
Exception.__init__(self, u'Unable to load module "%s": %s' % (module_name, msg))
self.module = module_name
class Module(object):
def __init__(self, package):
self.logger = getLogger('backend')
self.package = package
self.klass = None
for attrname in dir(self.package):
attr = getattr(self.package, attrname)
if isinstance(attr, type) and issubclass(attr, BaseBackend) and attr != BaseBackend:
self.klass = attr
if not self.klass:
raise ImportError('%s is not a backend (no BaseBackend class found)' % package)
@property
def name(self):
return self.klass.NAME
@property
def maintainer(self):
return '%s <%s>' % (self.klass.MAINTAINER, self.klass.EMAIL)
@property
def version(self):
return self.klass.VERSION
@property
def description(self):
return self.klass.DESCRIPTION
@property
def license(self):
return self.klass.LICENSE
@property
def config(self):
return self.klass.CONFIG
@property
def website(self):
if self.klass.BROWSER and hasattr(self.klass.BROWSER, 'DOMAIN') and self.klass.BROWSER.DOMAIN:
return '%s://%s' % (self.klass.BROWSER.PROTOCOL, self.klass.BROWSER.DOMAIN)
else:
return None
@property
def icon_path(self):
return self.klass.ICON
def iter_caps(self):
return self.klass.iter_caps()
def has_caps(self, *caps):
for c in caps:
if (isinstance(c, basestring) and c in [cap.__name__ for cap in self.iter_caps()]) or \
(type(c) == type and issubclass(self.klass, c)):
return True
return False
def create_instance(self, weboob, instance_name, config, storage):
backend_instance = self.klass(weboob, instance_name, config, storage, self.logger)
self.logger.debug(u'Created backend instance "%s" for backend "%s"' % (instance_name, self.name))
return backend_instance
class ModulesLoader(object):
def __init__(self):
self.loaded = {}
self.logger = getLogger('modules')
def get_or_load_module(self, module_name):
"""
Can raise a ModuleLoadError exception.
"""
if module_name not in self.loaded:
self.load_module(module_name)
return self.loaded[module_name]
def iter_existing_module_names(self):
try:
import weboob.backends
except ImportError:
return
for path in weboob.backends.__path__:
for root, dirs, files in os.walk(path):
if os.path.dirname( root ) == path and '__init__.py' in files:
s = os.path.basename( root )
yield s
def load_all(self):
for existing_module_name in self.iter_existing_module_names():
try:
self.load_module(existing_module_name)
except ModuleLoadError, e:
self.logger.warning(e)
def load_module(self, module_name):
try:
package_name = 'weboob.backends.%s' % module_name
module = Module(__import__(package_name, fromlist=[str(package_name)]))
except Exception, e:
if self.logger.level == logging.DEBUG:
self.logger.exception(e)
raise ModuleLoadError(module_name, e)
if module.name in self.loaded:
self.logger.debug('Module "%s" is already loaded from %s' % (module_name, module.package.__path__[0]))
return
self.loaded[module.name] = module
self.logger.debug('Loaded module "%s" from %s' % (module_name, module.package.__path__[0]))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/core/ouiboube.py 0000664 0000000 0000000 00000021707 11666415431 0025247 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
import os
from weboob.core.bcall import BackendsCall
from weboob.core.modules import ModulesLoader, ModuleLoadError
from weboob.core.backendscfg import BackendsConfig
from weboob.core.scheduler import Scheduler
from weboob.tools.backend import BaseBackend
from weboob.tools.log import getLogger
__all__ = ['Weboob']
class Weboob(object):
WORKDIR = os.path.join(os.path.expanduser('~'), '.weboob')
BACKENDS_FILENAME = 'backends'
def __init__(self, workdir=WORKDIR, backends_filename=None, scheduler=None, storage=None):
self.logger = getLogger('weboob')
self.workdir = workdir
self.backend_instances = {}
self.callbacks = {'login': lambda backend_name, value: None,
'captcha': lambda backend_name, image: None,
}
# Scheduler
if scheduler is None:
scheduler = Scheduler()
self.scheduler = scheduler
# Create WORKDIR
if not os.path.exists(self.workdir):
os.mkdir(self.workdir, 0700)
elif not os.path.isdir(self.workdir):
self.logger.warning(u'"%s" is not a directory' % self.workdir)
# Backends loader
self.modules_loader = ModulesLoader()
# Backend instances config
if not backends_filename:
backends_filename = os.path.join(self.workdir, self.BACKENDS_FILENAME)
elif not backends_filename.startswith('/'):
backends_filename = os.path.join(self.workdir, backends_filename)
self.backends_config = BackendsConfig(backends_filename)
# Storage
self.storage = storage
def __deinit__(self):
self.deinit()
def deinit(self):
self.unload_backends()
class LoadError(Exception):
def __init__(self, backend_name, exception):
Exception.__init__(self, unicode(exception))
self.backend_name = backend_name
def load_backends(self, caps=None, names=None, modules=None, storage=None, errors=None):
"""
Load backends.
@param caps [tuple(ICapBase)] load backends which implement all of caps
@param names [tuple(unicode)] load backends with instance name in list
@param modules [tuple(unicode)] load backends which module is in list
@param storage [IStorage] use the storage if specified
@param errors [list] if specified, store every errors in
@return [dict(str,BaseBackend)] return loaded backends
"""
loaded = {}
if storage is None:
storage = self.storage
for instance_name, module_name, params in self.backends_config.iter_backends():
if '_enabled' in params and not params['_enabled'].lower() in ('1', 'y', 'true', 'on', 'yes') or \
names is not None and instance_name not in names or \
modules is not None and module_name not in modules:
continue
module = None
try:
module = self.modules_loader.get_or_load_module(module_name)
except ModuleLoadError, e:
self.logger.error(e)
if module is None:
self.logger.warning(u'Backend "%s" is referenced in ~/.weboob/backends '
'configuration file, but was not found. '
'Hint: is it installed?' % module_name)
continue
if caps is not None and not module.has_caps(caps):
continue
if instance_name in self.backend_instances:
self.logger.warning(u'Oops, the backend "%s" is already loaded. Unload it before reloading...' % instance_name)
self.unload_backends(instance_name)
try:
backend_instance = module.create_instance(self, instance_name, params, storage)
except BaseBackend.ConfigError, e:
if errors is not None:
errors.append(self.LoadError(instance_name, e))
else:
self.backend_instances[instance_name] = loaded[instance_name] = backend_instance
return loaded
def unload_backends(self, names=None):
unloaded = {}
if isinstance(names, basestring):
names = [names]
elif names is None:
names = self.backend_instances.keys()
for name in names:
backend = self.backend_instances.pop(name)
with backend:
backend.deinit()
unloaded[backend.name] = backend
return unloaded
def get_backend(self, name, **kwargs):
"""
Get a backend from its name.
It raises a KeyError if not found. If you set the 'default' parameter,
the default value is returned instead.
"""
try:
return self.backend_instances[name]
except KeyError:
if 'default' in kwargs:
return kwargs['default']
else:
raise
def count_backends(self):
return len(self.backend_instances)
def iter_backends(self, caps=None):
"""
Iter on each backends.
Note: each backend is locked when it is returned.
@param caps Optional list of capabilities to select backends
@return iterator on selected backends.
"""
for name, backend in sorted(self.backend_instances.iteritems()):
if caps is None or backend.has_caps(caps):
with backend:
yield backend
def do(self, function, *args, **kwargs):
"""
Do calls on loaded backends with specified arguments, in separated
threads.
This function has two modes:
- If 'function' is a string, it calls the method with this name on
each backends with the specified arguments;
- If 'function' is a callable, it calls it in a separated thread with
the locked backend instance at first arguments, and \*args and
\*\*kwargs.
@param function backend's method name, or callable object
@param backends list of backends to iterate on
@param caps iterate on backends with this caps
@param condition a condition to validate to keep the result
@return the BackendsCall object (iterable)
"""
backends = self.backend_instances.values()
_backends = kwargs.pop('backends', None)
if _backends is not None:
if isinstance(_backends, BaseBackend):
backends = [_backends]
elif isinstance(_backends, basestring):
if len(_backends) > 0:
try:
backends = [self.backend_instances[_backends]]
except (ValueError,KeyError):
backends = []
elif isinstance(_backends, (list, tuple, set)):
backends = []
for backend in _backends:
if isinstance(backend, basestring):
try:
backends.append(self.backend_instances[backend])
except (ValueError,KeyError):
pass
else:
backends.append(backend)
else:
self.logger.warning(u'The "backends" value isn\'t supported: %r' % _backends)
if 'caps' in kwargs:
caps = kwargs.pop('caps')
backends = [backend for backend in backends if backend.has_caps(caps)]
condition = kwargs.pop('condition', None)
# The return value MUST BE the BackendsCall instance. Please never iterate
# here on this object, because caller might want to use other methods, like
# wait() on callback_thread().
# Thanks a lot.
return BackendsCall(backends, condition, function, *args, **kwargs)
def schedule(self, interval, function, *args):
return self.scheduler.schedule(interval, function, *args)
def repeat(self, interval, function, *args):
return self.scheduler.repeat(interval, function, *args)
def cancel(self, ev):
return self.scheduler.cancel(ev)
def want_stop(self):
return self.scheduler.want_stop()
def loop(self):
return self.scheduler.run()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/core/scheduler.py 0000664 0000000 0000000 00000007727 11666415431 0025422 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from threading import Timer, Event, RLock, _Timer
from weboob.tools.log import getLogger
__all__ = ['Scheduler']
class IScheduler(object):
def schedule(self, interval, function, *args):
raise NotImplementedError()
def repeat(self, interval, function, *args):
raise NotImplementedError()
def cancel(self, ev):
raise NotImplementedError()
def run(self):
raise NotImplementedError()
def want_stop(self):
raise NotImplementedError()
class RepeatedTimer(_Timer):
def run(self):
while not self.finished.isSet():
self.function(*self.args, **self.kwargs)
self.finished.wait(self.interval)
self.finished.set()
class Scheduler(IScheduler):
def __init__(self):
self.logger = getLogger('scheduler')
self.mutex = RLock()
self.stop_event = Event()
self.count = 0
self.queue = {}
def schedule(self, interval, function, *args):
return self._schedule(Timer, interval, self._schedule_callback, function, *args)
def repeat(self, interval, function, *args):
return self._schedule(RepeatedTimer, interval, self._repeat_callback, function, *args)
def _schedule(self, klass, interval, meta_func, function, *args):
if self.stop_event.isSet():
return
with self.mutex:
self.count += 1
self.logger.debug('function "%s" will be called in %s seconds' % (function.__name__, interval))
timer = klass(interval, meta_func, (self.count, interval, function, args))
self.queue[self.count] = timer
timer.start()
return self.count
def _schedule_callback(self, count, interval, function, args):
with self.mutex:
self.queue.pop(count)
return function(*args)
def _repeat_callback(self, count, interval, function, args):
function(*args)
with self.mutex:
try:
e = self.queue[count]
except KeyError:
return
else:
self.logger.debug('function "%s" will be called in %s seconds' % (function.__name__, e.interval))
def cancel(self, ev):
with self.mutex:
try:
e = self.queue.pop(ev)
except KeyError:
return False
e.cancel()
self.logger.debug('scheduled function "%s" is canceled' % e.function.__name__)
return True
def _wait_to_stop(self):
self.want_stop()
with self.mutex:
for e in self.queue.itervalues():
e.cancel()
e.join()
self.queue = {}
def run(self):
try:
while 1:
self.stop_event.wait(0.1)
except KeyboardInterrupt:
self._wait_to_stop()
raise
else:
self._wait_to_stop()
return True
def want_stop(self):
self.stop_event.set()
with self.mutex:
for t in self.queue.itervalues():
t.cancel()
# Contrary to _wait_to_stop(), don't call t.join
# because want_stop() have to be non-blocking.
self.queue = {}
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/ 0000775 0000000 0000000 00000000000 11666415431 0023265 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/__init__.py 0000664 0000000 0000000 00000000001 11666415431 0025365 0 ustar 00root root 0000000 0000000
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/ 0000775 0000000 0000000 00000000000 11666415431 0025570 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/__init__.py 0000664 0000000 0000000 00000000001 11666415431 0027670 0 ustar 00root root 0000000 0000000
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/base.py 0000664 0000000 0000000 00000032434 11666415431 0027062 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import logging
import optparse
from optparse import OptionGroup, OptionParser
import os
import sys
import tempfile
from weboob.capabilities.base import NotAvailable, NotLoaded
from weboob.core import Weboob, CallErrors
from weboob.core.backendscfg import BackendsConfig
from weboob.tools.config.iconfig import ConfigError
from weboob.tools.backend import ObjectNotAvailable
from weboob.tools.log import createColoredFormatter, getLogger
__all__ = ['BaseApplication']
class ApplicationStorage(object):
def __init__(self, name, storage):
self.name = name
self.storage = storage
def set(self, *args):
if self.storage:
return self.storage.set('applications', self.name, *args)
def delete(self, *args):
if self.storage:
return self.storage.delete('applications', self.name, *args)
def get(self, *args, **kwargs):
if self.storage:
return self.storage.get('applications', self.name, *args, **kwargs)
else:
return kwargs.get('default', None)
def load(self, default):
if self.storage:
return self.storage.load('applications', self.name, default)
def save(self):
if self.storage:
return self.storage.save('applications', self.name)
class BaseApplication(object):
"""
Base application.
This class can be herited to have some common code within weboob
applications.
"""
# ------ Class attributes --------------------------------------
# Application name
APPNAME = ''
# Configuration and work directory (default: ~/.weboob/)
CONFDIR = os.path.join(os.path.expanduser('~'), '.weboob')
# Default configuration dict (can only contain key/values)
CONFIG = {}
# Default storage tree
STORAGE = {}
# Synopsis
SYNOPSIS = 'Usage: %prog [-h] [-dqv] [-b backends] ...\n'
SYNOPSIS += ' %prog [--help] [--version]'
# Description
DESCRIPTION = None
# Version
VERSION = None
# Copyright
COPYRIGHT = None
# ------ Abstract methods --------------------------------------
def create_weboob(self):
return Weboob()
def _get_completions(self):
"""
Overload this method in subclasses if you want to enrich shell completion.
@return a set object
"""
return set()
def _handle_options(self):
"""
Overload this method in application type subclass
if you want to handle options defined in subclass constructor.
"""
pass
def add_application_options(self, group):
"""
Overload this method if your application needs extra options.
These options will be displayed in an option group.
"""
pass
def handle_application_options(self):
"""
Overload this method in your application if you want to handle options defined in add_application_options.
"""
pass
# ------ BaseApplication methods -------------------------------
def __init__(self, option_parser=None):
self.logger = getLogger(self.APPNAME)
self.weboob = self.create_weboob()
self.config = None
self.options = None
if option_parser is None:
self._parser = OptionParser(self.SYNOPSIS, version=self._get_optparse_version())
else:
self._parser = option_parser
if self.DESCRIPTION:
self._parser.description = self.DESCRIPTION
app_options = OptionGroup(self._parser, '%s Options' % self.APPNAME.capitalize())
self.add_application_options(app_options)
if len(app_options.option_list) > 0:
self._parser.add_option_group(app_options)
self._parser.add_option('-b', '--backends', help='what backend(s) to enable (comma separated)')
logging_options = OptionGroup(self._parser, 'Logging Options')
logging_options.add_option('-d', '--debug', action='store_true', help='display debug messages')
logging_options.add_option('-q', '--quiet', action='store_true', help='display only error messages')
logging_options.add_option('-v', '--verbose', action='store_true', help='display info messages')
logging_options.add_option('--logging-file', action='store', type='string', dest='logging_file', help='file to save logs')
logging_options.add_option('-a', '--save-responses', action='store_true', help='save every response')
self._parser.add_option_group(logging_options)
self._parser.add_option('--shell-completion', action='store_true', help=optparse.SUPPRESS_HELP)
def deinit(self):
self.weboob.want_stop()
self.weboob.deinit()
def create_storage(self, path=None, klass=None, localonly=False):
"""
Create a storage object.
@param path [str] an optional specific path.
@param klass [IStorage] what klass to instance.
@param localonly [bool] if True, do not set it on the Weboob object.
@return a IStorage object
"""
if klass is None:
from weboob.tools.storage import StandardStorage
klass = StandardStorage
if path is None:
path = os.path.join(self.CONFDIR, self.APPNAME + '.storage')
elif not path.startswith('/'):
path = os.path.join(self.CONFDIR, path)
storage = klass(path)
self.storage = ApplicationStorage(self.APPNAME, storage)
self.storage.load(self.STORAGE)
if not localonly:
self.weboob.storage = storage
return storage
def load_config(self, path=None, klass=None):
"""
Load a configuration file and get his object.
@param path [str] an optional specific path.
@param klass [IConfig] what klass to instance.
@return a IConfig object
"""
if klass is None:
from weboob.tools.config.iniconfig import INIConfig
klass = INIConfig
if path is None:
path = os.path.join(self.CONFDIR, self.APPNAME)
elif not path.startswith('/'):
path = os.path.join(self.CONFDIR, path)
self.config = klass(path)
self.config.load(self.CONFIG)
def main(self, argv):
"""
Main method
Called by run
"""
raise NotImplementedError()
def load_backends(self, caps=None, names=None, *args, **kwargs):
if names is None and self.options.backends:
names = self.options.backends.split(',')
loaded = self.weboob.load_backends(caps, names, *args, **kwargs)
if not loaded:
logging.info(u'No backend loaded')
return loaded
def _get_optparse_version(self):
version = None
if self.VERSION:
if self.COPYRIGHT:
version = '%s v%s %s' % (self.APPNAME, self.VERSION, self.COPYRIGHT)
else:
version = '%s v%s' % (self.APPNAME, self.VERSION)
return version
def _do_complete_obj(self, backend, fields, obj):
if fields:
try:
backend.fillobj(obj, fields)
except ObjectNotAvailable, e:
logging.warning(u'Could not retrieve required fields (%s): %s' % (','.join(fields), e))
for field in fields:
if getattr(obj, field) is NotLoaded:
setattr(obj, field, NotAvailable)
return obj
def _do_complete_iter(self, backend, count, fields, res):
for i, sub in enumerate(res):
if count and i == count:
break
sub = self._do_complete_obj(backend, fields, sub)
yield sub
def _do_complete(self, backend, count, selected_fields, function, *args, **kwargs):
assert count is None or count > 0
if callable(function):
res = function(backend, *args, **kwargs)
else:
res = getattr(backend, function)(*args, **kwargs)
if hasattr(res, '__iter__'):
return self._do_complete_iter(backend, count, selected_fields, res)
else:
return self._do_complete_obj(backend, selected_fields, res)
def bcall_error_handler(self, backend, error, backtrace):
"""
Handler for an exception inside the CallErrors exception.
This method can be overrided to support more exceptions types.
"""
print >>sys.stderr, u'Error(%s): %s' % (backend.name, error)
if logging.root.level == logging.DEBUG:
print >>sys.stderr, backtrace
def bcall_errors_handler(self, errors):
"""
Handler for the CallErrors exception.
"""
for backend, error, backtrace in errors.errors:
self.bcall_error_handler(backend, error, backtrace)
if logging.root.level != logging.DEBUG:
print >>sys.stderr, 'Use --debug option to print backtraces.'
def parse_args(self, args):
self.options, args = self._parser.parse_args(args)
if self.options.shell_completion:
items = set()
for option in self._parser.option_list:
if not option.help is optparse.SUPPRESS_HELP:
items.update(str(option).split('/'))
items.update(self._get_completions())
print ' '.join(items)
sys.exit(0)
if self.options.debug or self.options.save_responses:
level = logging.DEBUG
from weboob.tools.browser import StandardBrowser
StandardBrowser.DEBUG_MECHANIZE = True
# required to actually display or save the stuff
logger = logging.getLogger("mechanize")
logger.setLevel(logging.INFO)
elif self.options.verbose:
level = logging.INFO
elif self.options.quiet:
level = logging.ERROR
else:
level = logging.WARNING
logging.root.setLevel(level)
if self.options.save_responses:
responses_dirname = tempfile.mkdtemp(prefix='weboob_session_')
print >>sys.stderr, 'Debug data will be saved in this directory: %s' % responses_dirname
StandardBrowser.SAVE_RESPONSES = True
StandardBrowser.responses_dirname = responses_dirname
self.add_logging_file_handler(os.path.join(responses_dirname, 'debug.log'))
# file logger
if self.options.logging_file:
self.add_logging_file_handler(self.options.logging_file)
else:
# stdout logger
format = '%(asctime)s:%(levelname)s:%(name)s:%(filename)s:%(lineno)d:%(funcName)s %(message)s'
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(createColoredFormatter(sys.stdout, format))
logging.root.addHandler(handler)
self._handle_options()
self.handle_application_options()
return args
def add_logging_file_handler(self, filename):
try:
stream = open(filename, 'w')
except IOError, e:
self.logger.error('Unable to create the logging file: %s' % e)
sys.exit(1)
else:
format = '%(asctime)s:%(levelname)s:%(name)s:%(pathname)s:%(lineno)d:%(funcName)s %(message)s'
handler = logging.StreamHandler(stream)
handler.setFormatter(logging.Formatter(format))
logging.root.addHandler(handler)
@classmethod
def run(klass, args=None):
"""
This static method can be called to run the application.
It creates the application object, handles options, setups logging, calls
the main() method, and catches common exceptions.
You can't do anything after this call, as it *always* finishes with
a call to sys.exit().
For example:
>>> from weboob.application.myapplication import MyApplication
>>> MyApplication.run()
"""
if args is None:
args = [(sys.stdin.encoding and arg.decode(sys.stdin.encoding) or arg) for arg in sys.argv]
try:
app = klass()
except BackendsConfig.WrongPermissions, e:
print >>sys.stderr, e
sys.exit(1)
try:
try:
args = app.parse_args(args)
sys.exit(app.main(args))
except KeyboardInterrupt:
print >>sys.stderr, 'Program killed by SIGINT'
sys.exit(0)
except EOFError:
sys.exit(0)
except ConfigError, e:
print >>sys.stderr, 'Configuration error: %s' % e
sys.exit(1)
except CallErrors, e:
app.bcall_errors_handler(e)
sys.exit(1)
finally:
app.deinit()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/console.py 0000664 0000000 0000000 00000042350 11666415431 0027610 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz, Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from copy import copy
import getpass
import logging
import sys
import os
import locale
from weboob.capabilities.account import ICapAccount, Account, AccountRegisterError
from weboob.core.backendscfg import BackendAlreadyExists
from weboob.core.modules import ModuleLoadError
from weboob.tools.browser import BrowserUnavailable, BrowserIncorrectPassword
from weboob.tools.value import Value, ValueBool, ValueFloat, ValueInt
from .base import BaseApplication
__all__ = ['ConsoleApplication', 'BackendNotGiven']
class BackendNotGiven(Exception):
def __init__(self, id, backends):
self.id = id
self.backends = sorted(backends)
Exception.__init__(self, 'Please specify a backend to use for this argument (%s@backend_name). '
'Availables: %s.' % (id, ', '.join(name for name, backend in backends)))
class BackendNotFound(Exception):
pass
class ConsoleApplication(BaseApplication):
"""
Base application class for CLI applications.
"""
CAPS = None
# shell escape strings
if sys.platform == 'win32':
#workaround to disable bold
BOLD = ''
NC = '' # no color
else:
BOLD = '[1m'
NC = '[0m' # no color
stdin = sys.stdin
stdout = sys.stdout
def __init__(self, option_parser=None):
BaseApplication.__init__(self, option_parser)
self.weboob.callbacks['login'] = self.login_cb
self.enabled_backends = set()
def login_cb(self, backend_name, value):
return self.ask('[%s] Password' % backend_name, masked=True, default='', regexp=value.regexp)
def unload_backends(self, *args, **kwargs):
unloaded = self.weboob.unload_backends(*args, **kwargs)
for backend in unloaded.itervalues():
try:
self.enabled_backends.remove(backend)
except KeyError:
pass
return unloaded
def is_backend_loadable(self, backend):
return self.CAPS is None or self.caps_included(backend.iter_caps(), self.CAPS.__name__)
def load_backends(self, *args, **kwargs):
if 'errors' in kwargs:
errors = kwargs['errors']
else:
kwargs['errors'] = errors = []
ret = super(ConsoleApplication, self).load_backends(*args, **kwargs)
for err in errors:
print >>sys.stderr, 'Error(%s): %s' % (err.backend_name, err)
if self.ask('Do you want to reconfigure this backend?', default=True):
self.edit_backend(err.backend_name)
self.load_backends(names=[err.backend_name])
for name, backend in ret.iteritems():
self.enabled_backends.add(backend)
self.check_loaded_backends()
return ret
def check_loaded_backends(self, default_config=None):
while len(self.enabled_backends) == 0:
print 'Warning: there is currently no configured backend for %s' % self.APPNAME
if not os.isatty(sys.stdout.fileno()) or not self.ask('Do you want to configure backends?', default=True):
return False
self.prompt_create_backends(default_config)
return True
def prompt_create_backends(self, default_config=None):
self.weboob.modules_loader.load_all()
r = ''
while r != 'q':
backends = []
print '\nAvailable backends:'
for name, backend in sorted(self.weboob.modules_loader.loaded.iteritems()):
if not self.is_backend_loadable(backend):
continue
backends.append(name)
loaded = ' '
for bi in self.weboob.iter_backends():
if bi.NAME == name:
if loaded == ' ':
loaded = 'X'
elif loaded == 'X':
loaded = 2
else:
loaded += 1
print '%s%d)%s [%s] %s%-15s%s %s' % (self.BOLD, len(backends), self.NC, loaded,
self.BOLD, name, self.NC, backend.description)
print '%sq)%s --stop--\n' % (self.BOLD, self.NC)
r = self.ask('Select a backend to add (q to stop)', regexp='^(\d+|q)$')
if str(r).isdigit():
i = int(r) - 1
if i < 0 or i >= len(backends):
print >>sys.stderr, 'Error: %s is not a valid choice' % r
continue
name = backends[i]
try:
inst = self.add_backend(name, default_config)
if inst:
self.load_backends(names=[inst])
except (KeyboardInterrupt, EOFError):
print '\nAborted.'
print 'Right right!'
def _handle_options(self):
self.load_default_backends()
def load_default_backends(self):
"""
By default loads all backends.
Applications can overload this method to restrict backends loaded.
"""
self.load_backends(self.CAPS)
@classmethod
def run(klass, args=None):
try:
super(ConsoleApplication, klass).run(args)
except BackendNotFound, e:
print 'Error: Backend "%s" not found.' % e
sys.exit(1)
def do(self, function, *args, **kwargs):
if not 'backends' in kwargs:
kwargs['backends'] = self.enabled_backends
return self.weboob.do(function, *args, **kwargs)
def parse_id(self, _id, unique_backend=False):
try:
_id, backend_name = _id.rsplit('@', 1)
except ValueError:
backend_name = None
backends = [(b.name, b) for b in self.enabled_backends]
if unique_backend and not backend_name:
if len(backends) == 1:
backend_name = backends[0][0]
else:
raise BackendNotGiven(_id, backends)
if backend_name is not None and not backend_name in dict(backends):
raise BackendNotFound(backend_name)
return _id, backend_name
def caps_included(self, modcaps, caps):
modcaps = [x.__name__ for x in modcaps]
if not isinstance(caps, (list,set,tuple)):
caps = (caps,)
for cap in caps:
if not cap in modcaps:
return False
return True
# user interaction related methods
def register_backend(self, name, ask_add=True):
try:
backend = self.weboob.modules_loader.get_or_load_module(name)
except ModuleLoadError, e:
backend = None
if not backend:
print >>sys.stderr, 'Backend "%s" does not exist.' % name
return 1
if not backend.has_caps(ICapAccount) or backend.klass.ACCOUNT_REGISTER_PROPERTIES is None:
print >>sys.stderr, 'You can\'t register a new account with %s' % name
return 1
account = Account()
account.properties = {}
if backend.website:
website = 'on website %s' % backend.website
else:
website = 'with backend %s' % backend.name
while 1:
asked_config = False
for key, prop in backend.klass.ACCOUNT_REGISTER_PROPERTIES.iteritems():
if not asked_config:
asked_config = True
print 'Configuration of new account %s' % website
print '-----------------------------%s' % ('-' * len(website))
p = copy(prop)
p.set(self.ask(prop, default=account.properties[key].get() if (key in account.properties) else prop.default))
account.properties[key] = p
if asked_config:
print '-----------------------------%s' % ('-' * len(website))
try:
backend.klass.register_account(account)
except AccountRegisterError, e:
print u'%s' % e
if self.ask('Do you want to try again?', default=True):
continue
else:
return None
else:
break
backend_config = {}
for key, value in account.properties.iteritems():
if key in backend.config:
backend_config[key] = value.get()
if ask_add and self.ask('Do you want to add the new register account?', default=True):
return self.add_backend(name, backend_config, ask_register=False)
return backend_config
def edit_backend(self, name, params=None):
return self.add_backend(name, params, True)
def add_backend(self, name, params=None, edit=False, ask_register=True):
if params is None:
params = {}
backend = None
config = None
if not edit:
try:
backend = self.weboob.modules_loader.get_or_load_module(name)
config = backend.config
except ModuleLoadError:
backend = None
else:
bname, items = self.weboob.backends_config.get_backend(name)
try:
backend = self.weboob.modules_loader.get_or_load_module(bname)
except ModuleLoadError:
backend = None
else:
items.update(params)
params = items
config = backend.config.load(self.weboob, bname, name, params, nofail=True)
if not backend:
print >>sys.stderr, 'Backend "%s" does not exist. Hint: use the "backends" command.' % name
return 1
# ask for params non-specified on command-line arguments
asked_config = False
for key, value in config.iteritems():
if not asked_config:
asked_config = True
print 'Configuration of backend'
print '------------------------'
if key not in params or edit:
params[key] = self.ask(value, default=params[key] if (key in params) else value.default)
else:
print u' [%s] %s: %s' % (key, value.description, '(masked)' if value.masked else params[key])
if asked_config:
print '------------------------'
while not edit and self.weboob.backends_config.backend_exists(name):
print >>sys.stderr, 'Backend instance "%s" already exists in "%s"' % (name, self.weboob.backends_config.confpath)
if not self.ask('Add new instance of "%s" backend?' % backend.name, default=False):
return 1
name = self.ask('Please give new instance name (could be "%s_1")' % backend.name, regexp=r'^[\w\-_]+$')
try:
config = config.load(self.weboob, backend.name, name, params, nofail=True)
for key, value in params.iteritems():
config[key].set(value)
config.save(edit=edit)
print 'Backend "%s" successfully added.' % name
return name
except BackendAlreadyExists:
print >>sys.stderr, 'Instance "%s" already exists.' % name
return 1
def ask(self, question, default=None, masked=False, regexp=None, choices=None):
"""
Ask a question to user.
@param question text displayed (str)
@param default optional default value (str)
@param masked if True, do not show typed text (bool)
@param regexp text must match this regexp (str)
@param choices choices to do (list)
@return entered text by user (str)
"""
if isinstance(question, Value):
v = copy(question)
if default:
v.default = default
if masked:
v.masked = masked
if regexp:
v.regexp = regexp
if choices:
v.choices = choices
else:
if isinstance(default, bool):
klass = ValueBool
elif isinstance(default, float):
klass = ValueFloat
elif isinstance(default, (int,long)):
klass = ValueInt
else:
klass = Value
v = klass(label=question, default=default, masked=masked, regexp=regexp, choices=choices)
question = v.label
if v.id:
question = u'[%s] %s' % (v.id, question)
aliases = {}
if isinstance(v, ValueBool):
question = u'%s (%s/%s)' % (question, 'Y' if v.default else 'y', 'n' if v.default else 'N')
elif v.choices:
tiny = True
for key in v.choices.iterkeys():
if len(key) > 5 or ' ' in key:
tiny = False
break
if tiny:
question = u'%s (%s)' % (question, '/'.join((s.upper() if s == v.default else s)
for s in (v.choices.iterkeys())))
else:
for n, (key, value) in enumerate(v.choices.iteritems()):
print '%s%2d)%s %s' % (self.BOLD, n + 1, self.NC, value)
aliases[str(n + 1)] = key
question = u'%s (choose in list)' % question
elif default not in (None, '') and not v.masked:
question = u'%s [%s]' % (question, v.default)
if v.masked:
question = u'%s (hidden input)' % question
question += ': '
while True:
if v.masked:
if sys.platform == 'win32':
line = getpass.getpass(str(question))
else:
line = getpass.getpass(question)
else:
self.stdout.write(question.encode(sys.stdout.encoding or locale.getpreferredencoding()))
self.stdout.flush()
line = self.stdin.readline()
if len(line) == 0:
raise EOFError()
else:
line = line.rstrip('\r\n')
if not line and v.default is not None:
line = v.default
if isinstance(line, str):
line = line.decode('utf-8')
if line in aliases:
line = aliases[line]
try:
v.set(line)
except ValueError, e:
print >>sys.stderr, 'Error: %s' % e
else:
break
return v.get()
def acquire_input(self):
if sys.stdin.isatty():
print 'Reading content from stdin... Type ctrl-D ' \
'from an empty line to stop.'
text = sys.stdin.read()
return text.decode(sys.stdin.encoding or locale.getpreferredencoding())
def bcall_error_handler(self, backend, error, backtrace):
"""
Handler for an exception inside the CallErrors exception.
This method can be overrided to support more exceptions types.
"""
if isinstance(error, BrowserIncorrectPassword):
msg = unicode(error)
if not msg:
msg = 'invalid login/password.'
print >>sys.stderr, 'Error(%s): %s' % (backend.name, msg)
if self.ask('Do you want to reconfigure this backend?', default=True):
self.unload_backends(names=[backend.name])
self.edit_backend(backend.name)
self.load_backends(names=[backend.name])
elif isinstance(error, BrowserUnavailable):
msg = unicode(error)
if not msg:
msg = 'website is unavailable.'
print >>sys.stderr, u'Error(%s): %s' % (backend.name, msg)
elif isinstance(error, NotImplementedError):
print >>sys.stderr, u'Error(%s): this feature is not supported yet by this backend.' % backend.name
print >>sys.stderr, u' %s To help the maintainer of this backend implement this feature,' % (' ' * len(backend.name))
print >>sys.stderr, u' %s please contact: %s <%s>' % (' ' * len(backend.name), backend.MAINTAINER, backend.EMAIL)
else:
print >>sys.stderr, u'Error(%s): %s' % (backend.name, error)
if logging.root.level == logging.DEBUG:
print >>sys.stderr, backtrace
else:
return True
def bcall_errors_handler(self, errors, debugmsg='Use --debug option to print backtraces'):
"""
Handler for the CallErrors exception.
"""
ask_debug_mode = False
for backend, error, backtrace in errors.errors:
if self.bcall_error_handler(backend, error, backtrace):
ask_debug_mode = True
if ask_debug_mode:
print >>sys.stderr, debugmsg
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/formatters/ 0000775 0000000 0000000 00000000000 11666415431 0027756 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/formatters/__init__.py0000664 0000000 0000000 00000000001 11666415431 0032056 0 ustar 00root root 0000000 0000000
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/formatters/csv.py 0000664 0000000 0000000 00000002427 11666415431 0031130 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .iformatter import IFormatter
__all__ = ['CSVFormatter']
class CSVFormatter(IFormatter):
def __init__(self, field_separator=u';'):
IFormatter.__init__(self)
self.field_separator = field_separator
self.count = 0
def flush(self):
pass
def format_dict(self, item):
result = u''
if self.count == 0:
result += self.field_separator.join(item.iterkeys()) + '\n'
self.count += 1
result += self.field_separator.join(unicode(v) for v in item.itervalues())
return result
iformatter.py 0000664 0000000 0000000 00000014372 11666415431 0032434 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/formatters # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
import os
import sys
import subprocess
if sys.platform == 'win32':
import WConio
try:
import tty, termios
except ImportError:
PROMPT = '--Press return to continue--'
def readch():
return sys.stdin.readline()
else:
PROMPT = '--Press a key to continue--'
def readch():
fd = sys.stdin.fileno()
old_settings = termios.tcgetattr(fd)
tty.setraw(fd)
try:
c = sys.stdin.read(1)
# XXX do not read magic number
if c == '\x03':
raise KeyboardInterrupt()
return c
finally:
termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
from weboob.capabilities.base import CapBaseObject, FieldNotFound
from weboob.tools.ordereddict import OrderedDict
from weboob.tools.application.console import ConsoleApplication
__all__ = ['IFormatter', 'MandatoryFieldsNotFound']
class MandatoryFieldsNotFound(Exception):
def __init__(self, missing_fields):
Exception.__init__(self, u'Mandatory fields not found: %s.' % ', '.join(missing_fields))
class IFormatter(object):
MANDATORY_FIELDS = None
def get_bold(self):
if self.outfile != sys.stdout:
return ''
else:
return ConsoleApplication.BOLD
def get_nc(self):
if self.outfile != sys.stdout:
return ''
else:
return ConsoleApplication.NC
BOLD = property(get_bold)
NC = property(get_nc)
def __init__(self, display_keys=True, display_header=True, return_only=False, outfile=sys.stdout):
self.display_keys = display_keys
self.display_header = display_header
self.return_only = return_only
self.interactive = False
self.print_lines = 0
self.termrows = 0
self.outfile = outfile
# XXX if stdin is not a tty, it seems that the command fails.
if os.isatty(sys.stdout.fileno()) and os.isatty(sys.stdin.fileno()):
if sys.platform == 'win32':
self.termrows = WConio.gettextinfo()[8]
else:
self.termrows = int( subprocess.Popen('stty size', shell=True, stdout=subprocess.PIPE).communicate()[0].split()[0])
def after_format(self, formatted):
if self.outfile != sys.stdout:
with open(self.outfile, "a+") as outfile:
outfile.write(formatted.encode('utf-8'))
else:
for line in formatted.split('\n'):
if self.termrows and (self.print_lines + 1) >= self.termrows:
self.outfile.write(PROMPT)
self.outfile.flush()
readch()
self.outfile.write('\b \b' * len(PROMPT))
self.print_lines = 0
if isinstance(line, unicode):
line = line.encode('utf-8')
print line
self.print_lines += 1
def build_id(self, v, backend_name):
return u'%s@%s' % (unicode(v), backend_name)
def flush(self):
raise NotImplementedError()
def format(self, obj, selected_fields=None):
"""
Format an object to be human-readable.
An object has fields which can be selected.
If the object provides an iter_fields() method, the formatter will
call it. It can be used to specify the fields order.
@param obj [object] object to format
@param selected_fields [tuple] fields to display. If None, all fields are selected
@return a string of the formatted object
"""
assert isinstance(obj, (dict, CapBaseObject, tuple)), 'Object is unexpected type "%r"' % obj
if isinstance(obj, dict):
item = obj
elif isinstance(obj, tuple):
item = OrderedDict([(k, v) for k, v in obj])
else:
item = self.to_dict(obj, selected_fields)
if item is None:
return None
if self.MANDATORY_FIELDS:
missing_fields = set(self.MANDATORY_FIELDS) - set(item.keys())
if missing_fields:
raise MandatoryFieldsNotFound(missing_fields)
formatted = self.format_dict(item=item)
if formatted:
self.after_format(formatted)
return formatted
def format_dict(self, item):
"""
Format a dict to be human-readable. The dict is already simplified
if user provides selected fields.
Called by format().
This method has to be overridden in child classes.
@param item [dict] item to format
@return a string of the formatted dict
"""
raise NotImplementedError()
def set_header(self, string):
if self.display_header:
print string.encode('utf-8')
def to_dict(self, obj, selected_fields=None):
def iter_select(d):
if selected_fields is None or '*' in selected_fields:
fields = d.iterkeys()
else:
fields = selected_fields
for key in fields:
try:
value = d[key]
except KeyError:
raise FieldNotFound(obj, key)
yield key, value
def iter_decorate(d):
for key, value in d:
if key == 'id' and obj.backend is not None:
value = self.build_id(value, obj.backend)
yield key, value
fields_iterator = obj.iter_fields()
d = OrderedDict(iter_decorate(fields_iterator))
return OrderedDict((k, v) for k, v in iter_select(d))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/formatters/load.py 0000664 0000000 0000000 00000004730 11666415431 0031253 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz, Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
__all__ = ['FormattersLoader', 'FormatterLoadError']
class FormatterLoadError(Exception):
pass
class FormattersLoader(object):
BUILTINS = ['htmltable', 'multiline', 'simple', 'table', 'csv', 'webkit']
def __init__(self):
self.formatters = {}
def register_formatter(self, name, klass):
self.formatters[name] = klass
def get_available_formatters(self):
l = set(self.formatters.iterkeys())
l = l.union(self.BUILTINS)
l = list(l)
l.sort()
return l
def build_formatter(self, name):
if not name in self.formatters:
try:
self.formatters[name] = self.load_builtin_formatter(name)
except ImportError, e:
FormattersLoader.BUILTINS.remove(name)
raise FormatterLoadError('Unable to load formatter "%s": %s' % (name, e))
return self.formatters[name]()
def load_builtin_formatter(self, name):
if not name in self.BUILTINS:
raise FormatterLoadError('Formatter "%s" does not exist' % name)
if name == 'htmltable':
from .table import HTMLTableFormatter
return HTMLTableFormatter
elif name == 'table':
from .table import TableFormatter
return TableFormatter
elif name == 'simple':
from .simple import SimpleFormatter
return SimpleFormatter
elif name == 'multiline':
from .multiline import MultilineFormatter
return MultilineFormatter
elif name == 'webkit':
from .webkit import WebkitGtkFormatter
return WebkitGtkFormatter
elif name == 'csv':
from .csv import CSVFormatter
return CSVFormatter
multiline.py 0000664 0000000 0000000 00000003062 11666415431 0032254 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/formatters # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.base import NotLoaded, NotAvailable
from .iformatter import IFormatter
__all__ = ['MultilineFormatter']
class MultilineFormatter(IFormatter):
def __init__(self, key_value_separator=u': ', after_item=u'\n'):
IFormatter.__init__(self)
self.key_value_separator = key_value_separator
self.after_item = after_item
def flush(self):
pass
def format_dict(self, item):
result = u'\n'.join(u'%s%s' % (
(u'%s%s' % (k, self.key_value_separator) if self.display_keys else ''), v)
for k, v in item.iteritems() if (v is not NotLoaded and v is not NotAvailable))
if len(item) > 1:
result += self.after_item
return result
def set_header(self, string):
if self.display_header:
print string.encode('utf-8')
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/formatters/simple.py 0000664 0000000 0000000 00000002426 11666415431 0031625 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .iformatter import IFormatter
__all__ = ['SimpleFormatter']
class SimpleFormatter(IFormatter):
def __init__(self, field_separator=u'\t', key_value_separator=u'='):
IFormatter.__init__(self)
self.field_separator = field_separator
self.key_value_separator = key_value_separator
def flush(self):
pass
def format_dict(self, item):
return self.field_separator.join(u'%s%s' % (
(u'%s%s' % (k, self.key_value_separator) if self.display_keys else ''), v)
for k, v in item.iteritems())
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/formatters/table.py 0000664 0000000 0000000 00000005630 11666415431 0031423 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from prettytable import PrettyTable
from weboob.capabilities.base import NotLoaded, NotAvailable
from .iformatter import IFormatter
__all__ = ['TableFormatter', 'HTMLTableFormatter']
class TableFormatter(IFormatter):
HTML = False
def __init__(self, display_keys=True, return_only=False):
IFormatter.__init__(self, display_keys=display_keys, return_only=return_only)
self.queue = []
self.keys = None
self.header = None
def after_format(self, formatted):
if self.keys is None:
self.keys = formatted.keys()
self.queue.append(formatted.values())
def flush(self):
if len(self.queue) == 0:
return
queue = [() for i in xrange(len(self.queue))]
column_headers = []
# Do not display columns when all values are NotLoaded or NotAvailable
for i in xrange(len(self.keys)):
available = False
for line in self.queue:
if line[i] is not NotLoaded and line[i] is not NotAvailable and line[i] is not None:
available = True
break
if available:
column_headers.append(self.keys[i].capitalize().replace('_', ' '))
for j in xrange(len(self.queue)):
queue[j] += (self.queue[j][i],)
s = ''
if self.display_header and self.header:
if self.HTML:
s+= '
%s
' % self.header
else:
s += self.header
s += "\n"
table = PrettyTable(list(column_headers))
for column_header in column_headers:
table.set_field_align(column_header, 'l')
for line in queue:
table.add_row(line)
if self.HTML:
s += table.get_html_string()
else:
s += table.get_string()
self.queue = []
if self.return_only:
return s
else:
print s.encode('utf-8')
def format_dict(self, item):
# format is done in self.flush() by prettytable
return item
def set_header(self, string):
self.header = string
class HTMLTableFormatter(TableFormatter):
HTML = True
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/formatters/webkit/ 0000775 0000000 0000000 00000000000 11666415431 0031243 5 ustar 00root root 0000000 0000000 __init__.py 0000664 0000000 0000000 00000001456 11666415431 0033303 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/formatters/webkit # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .webkitgtk import WebkitGtkFormatter
__all__ = ['WebkitGtkFormatter']
webkitgtk.py 0000664 0000000 0000000 00000005267 11666415431 0033543 0 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/formatters/webkit # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import os
import gtk
import webkit
from weboob.tools.application.javascript import get_javascript
from ..table import HTMLTableFormatter
__all__ = ['WebkitGtkFormatter']
class WebBrowser(gtk.Window):
def __init__(self):
gtk.Window.__init__(self)
self.connect('destroy', gtk.main_quit)
self.set_default_size(800, 600)
self.web_view = webkit.WebView()
sw = gtk.ScrolledWindow()
sw.add(self.web_view)
self.add(sw)
self.show_all()
class WebkitGtkFormatter(HTMLTableFormatter):
def __init__(self):
HTMLTableFormatter.__init__(self, return_only=True)
def flush(self):
table_string = HTMLTableFormatter.flush(self)
js_filepaths = []
js_filepaths.append(get_javascript('jquery'))
js_filepaths.append(get_javascript('tablesorter'))
scripts = ['' % js_filepath for js_filepath in js_filepaths]
html_string_params = dict(table=table_string)
if scripts:
html_string_params['scripts'] = ''.join(scripts)
html_string = """
%(scripts)s
%(table)s
""" % html_string_params
web_browser = WebBrowser()
web_browser.web_view.load_html_string(html_string, 'file://%s' % os.path.abspath(os.getcwd()))
gtk.main()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/javascript.py 0000664 0000000 0000000 00000003444 11666415431 0030315 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import os
__all__ = ['get_javascript']
def get_javascript(name, load_order=('local', 'web'), minified=True):
if name == 'jquery':
for src in load_order:
if src == 'local':
# try Debian paths
if minified:
filepath = '/usr/share/javascript/jquery/jquery.min.js'
else:
filepath = '/usr/share/javascript/jquery/jquery.js'
if os.path.exists(filepath):
return filepath
elif src == 'web':
# return Google-hosted URLs
if minified:
return 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js'
else:
return 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.js'
elif name == 'tablesorter':
if 'web' in load_order:
if minified:
return 'http://tablesorter.com/jquery.tablesorter.min.js'
else:
return 'http://tablesorter.com/jquery.tablesorter.js'
return None
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/media_player.py 0000664 0000000 0000000 00000011701 11666415431 0030575 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz, Romain Bignon, John Obbele
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import os
from subprocess import Popen, PIPE
from weboob.tools.log import getLogger
__all__ = ['InvalidMediaPlayer', 'MediaPlayer', 'MediaPlayerNotFound']
PLAYERS = (
('mplayer', '-'),
('vlc', '-'),
('parole', 'fd://0'),
('totem', 'fd://0'),
('xine', 'stdin:/'),
)
class MediaPlayerNotFound(Exception):
def __init__(self):
Exception.__init__(self, u'No media player found on this system. Please install one of them: %s.' % \
', '.join(player[0] for player in PLAYERS))
class InvalidMediaPlayer(Exception):
def __init__(self, player_name):
Exception.__init__(self, u'Invalid media player: %s. Valid media players: %s.' % (
player_name, ', '.join(player[0] for player in PLAYERS)))
class MediaPlayer(object):
"""
Black magic invoking a media player to this world.
Presently, due to strong disturbances in the holidays of the ether
world, the media player used is chosen from a static list of
programs. See PLAYERS for more information.
"""
def __init__(self, logger=None):
self.logger = getLogger('mediaplayer', logger)
def guess_player_name(self):
for player_name in [player[0] for player in PLAYERS]:
if self._find_in_path(os.environ['PATH'], player_name):
return player_name
return None
def play(self, media, player_name=None):
"""
Play a media object, using programs from the PLAYERS list.
This function dispatch calls to either _play_default or
_play_rtmp for special rtmp streams using SWF verification.
"""
player_names = [player[0] for player in PLAYERS]
if player_name:
if player_name not in player_names:
raise InvalidMediaPlayer(player_name)
else:
self.logger.debug(u'No media player given. Using the first available from: %s.' % \
', '.join(player_names))
player_name = self.guess_player_name()
if player_name is None:
raise MediaPlayerNotFound()
if media.url.find('rtmp') == 0:
self._play_rtmp(media, player_name)
else:
self._play_default(media, player_name)
def _play_default(self, media, player_name):
"""
Play media.url with the media player.
"""
print 'Invoking "%s %s".' % (player_name, media.url)
os.spawnlp(os.P_WAIT, player_name, player_name, media.url)
def _play_rtmp(self, media, player_name):
"""
Download data with rtmpdump and pipe them to a media player.
You need a working version of rtmpdump installed and the SWF
object url in order to comply with SWF verification requests
from the server. The last one is retrieved from the non-standard
non-API compliant 'swf_player' attribute of the 'media' object.
"""
if not self._find_in_path(os.environ['PATH'], 'rtmpdump'):
self.logger.warning('"rtmpdump" binary not found')
return self._play_default(media, player_name)
media_url = media.url
try:
player_url = media.swf_player
if media.swf_player:
rtmp = 'rtmpdump -r %s --swfVfy %s' % (media_url, player_url)
else:
rtmp = 'rtmpdump -r %s' % media_url
except AttributeError:
self.logger.warning('Your media object does not have a "swf_player" attribute. SWF verification will be '
'disabled and may prevent correct media playback.')
return self._play_default(media, player_name)
rtmp += ' --quiet'
args = None
for (binary, stdin_args) in PLAYERS:
if binary == player_name:
args = stdin_args
assert args is not None
print ':: Streaming from %s' % media_url
print ':: to %s %s' % (player_name, args)
print ':: %s' % rtmp
p1 = Popen(rtmp.split(), stdout=PIPE)
Popen([player_name, args], stdin=p1.stdout, stderr=PIPE)
def _find_in_path(self,path, filename):
for i in path.split(':'):
if os.path.exists('/'.join([i, filename])):
return True
return False
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/qt/ 0000775 0000000 0000000 00000000000 11666415431 0026214 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/qt/Makefile 0000664 0000000 0000000 00000000264 11666415431 0027656 0 ustar 00root root 0000000 0000000 UI_FILES = $(wildcard *.ui)
UI_PY_FILES = $(UI_FILES:%.ui=%_ui.py)
PYUIC = pyuic4
all: $(UI_PY_FILES)
%_ui.py: %.ui
$(PYUIC) -o $@ $^
clean:
rm -f *.pyc
rm -f $(UI_PY_FILES)
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/qt/__init__.py 0000664 0000000 0000000 00000000301 11666415431 0030317 0 ustar 00root root 0000000 0000000 from .qt import QtApplication, QtMainWindow, QtDo, HTMLDelegate
from .backendcfg import BackendCfg
__all__ = ['QtApplication', 'QtMainWindow', 'QtDo', 'HTMLDelegate',
'BackendCfg']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/qt/backendcfg.py 0000664 0000000 0000000 00000036734 11666415431 0030652 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from PyQt4.QtGui import QDialog, QTreeWidgetItem, QLabel, QFormLayout, \
QMessageBox, QPixmap, QImage, QIcon, QHeaderView, \
QListWidgetItem, QTextDocument, QVBoxLayout, \
QDialogButtonBox
from PyQt4.QtCore import SIGNAL, Qt, QVariant, QUrl
import re
from logging import warning
from weboob.core.modules import ModuleLoadError
from weboob.core.backendscfg import BackendAlreadyExists
from weboob.capabilities.account import ICapAccount, Account, AccountRegisterError
from weboob.tools.application.qt.backendcfg_ui import Ui_BackendCfg
from weboob.tools.ordereddict import OrderedDict
from .qt import QtValue
class BackendCfg(QDialog):
def __init__(self, weboob, caps=None, parent=None):
QDialog.__init__(self, parent)
self.ui = Ui_BackendCfg()
self.ui.setupUi(self)
self.ui.configuredBackendsList.sortByColumn(0, Qt.AscendingOrder)
self.to_unload = set()
self.to_load = set()
self.weboob = weboob
self.caps = caps
self.config_widgets = {}
# This attribute is set when itemChanged it called, because when
# a backend is enabled/disabled, we don't want to display its config
# frame, and the itemClicked event is always emit just after a
# itemChanged event.
# is_enabling is a counter to prevent race conditions.
self.is_enabling = 0
self.weboob.modules_loader.load_all()
self.ui.configuredBackendsList.header().setResizeMode(QHeaderView.ResizeToContents)
self.ui.configFrame.hide()
self.icon_cache = {}
for name, backend in reversed(sorted(list(self.weboob.modules_loader.loaded.iteritems()))):
if not self.caps or backend.has_caps(*self.caps):
item = QListWidgetItem(name.capitalize())
if backend.icon_path:
item.setIcon(self.get_icon_cache(backend.icon_path))
self.ui.backendsList.addItem(item)
self.loadConfiguredBackendsList()
self.connect(self.ui.configuredBackendsList, SIGNAL('itemClicked(QTreeWidgetItem *, int)'),
self.configuredBackendClicked)
self.connect(self.ui.configuredBackendsList, SIGNAL('itemChanged(QTreeWidgetItem *, int)'),
self.configuredBackendEnabled)
self.connect(self.ui.backendsList, SIGNAL('itemSelectionChanged()'), self.backendSelectionChanged)
self.connect(self.ui.proxyBox, SIGNAL('toggled(bool)'), self.proxyEditEnabled)
self.connect(self.ui.addButton, SIGNAL('clicked()'), self.addEvent)
self.connect(self.ui.removeButton, SIGNAL('clicked()'), self.removeEvent)
self.connect(self.ui.registerButton, SIGNAL('clicked()'), self.registerEvent)
self.connect(self.ui.configButtonBox, SIGNAL('accepted()'), self.acceptBackend)
self.connect(self.ui.configButtonBox, SIGNAL('rejected()'), self.rejectBackend)
def get_icon_cache(self, path):
if not path in self.icon_cache:
img = QImage(path)
self.icon_cache[path] = QIcon(QPixmap.fromImage(img))
return self.icon_cache[path]
def loadConfiguredBackendsList(self):
self.ui.configuredBackendsList.clear()
for instance_name, name, params in self.weboob.backends_config.iter_backends():
try:
backend = self.weboob.modules_loader.get_or_load_module(name)
except ModuleLoadError:
backend = None
if not backend or self.caps and not backend.has_caps(*self.caps):
continue
item = QTreeWidgetItem(None, [instance_name, name])
item.setCheckState(0, Qt.Checked if params.get('_enabled', '1').lower() in ('1', 'y', 'true') \
else Qt.Unchecked)
if backend.icon_path:
item.setIcon(0, self.get_icon_cache(backend.icon_path))
self.ui.configuredBackendsList.addTopLevelItem(item)
def configuredBackendEnabled(self, item, col):
self.is_enabling += 1
instname = unicode(item.text(0))
bname = unicode(item.text(1))
if item.checkState(0) == Qt.Checked:
self.to_load.add(instname)
enabled = '1'
else:
self.to_unload.add(instname)
try:
self.to_load.remove(instname)
except KeyError:
pass
enabled = '0'
self.weboob.backends_config.edit_backend(instname, bname, {'_enabled': enabled})
def configuredBackendClicked(self, item, col):
if self.is_enabling:
self.is_enabling -= 1
return
bname = unicode(item.text(0))
self.editBackend(bname)
def addEvent(self):
self.editBackend()
def removeEvent(self):
item = self.ui.configuredBackendsList.currentItem()
if not item:
return
bname = unicode(item.text(0))
reply = QMessageBox.question(self, self.tr('Remove a backend'),
unicode(self.tr("Are you sure you want to remove the backend '%s'?")) % bname,
QMessageBox.Yes|QMessageBox.No)
if reply != QMessageBox.Yes:
return
self.weboob.backends_config.remove_backend(bname)
self.to_unload.add(bname)
try:
self.to_load.remove(bname)
except KeyError:
pass
self.ui.configFrame.hide()
self.loadConfiguredBackendsList()
def editBackend(self, name=None):
self.ui.registerButton.hide()
self.ui.configFrame.show()
if name is not None:
bname, params = self.weboob.backends_config.get_backend(name)
items = self.ui.backendsList.findItems(bname, Qt.MatchFixedString)
if not items:
warning('Backend not found')
else:
self.ui.backendsList.setCurrentItem(items[0])
self.ui.backendsList.setEnabled(False)
self.ui.nameEdit.setText(name)
self.ui.nameEdit.setEnabled(False)
if '_proxy' in params:
self.ui.proxyBox.setChecked(True)
self.ui.proxyEdit.setText(params.pop('_proxy'))
else:
self.ui.proxyBox.setChecked(False)
self.ui.proxyEdit.clear()
params.pop('_enabled', None)
backend = self.weboob.modules_loader.loaded[bname]
for key, value in backend.config.load(self.weboob, bname, name, params, nofail=True).iteritems():
try:
l, widget = self.config_widgets[key]
except KeyError:
warning('Key "%s" is not found' % key)
else:
widget.set_value(value)
else:
self.ui.nameEdit.clear()
self.ui.nameEdit.setEnabled(True)
self.ui.proxyBox.setChecked(False)
self.ui.proxyEdit.clear()
self.ui.backendsList.setEnabled(True)
self.ui.backendsList.setCurrentRow(-1)
def backendSelectionChanged(self):
for key, (label, value) in self.config_widgets.iteritems():
label.hide()
value.hide()
self.ui.configLayout.removeWidget(label)
self.ui.configLayout.removeWidget(value)
label.deleteLater()
value.deleteLater()
self.config_widgets = {}
self.ui.backendInfo.clear()
selection = self.ui.backendsList.selectedItems()
if not selection:
return
backend = self.weboob.modules_loader.loaded[unicode(selection[0].text()).lower()]
if backend.icon_path:
img = QImage(backend.icon_path)
self.ui.backendInfo.document().addResource(QTextDocument.ImageResource, QUrl('mydata://logo.png'),
QVariant(img))
if not backend.name in [n for n, ign, ign2 in self.weboob.backends_config.iter_backends()]:
self.ui.nameEdit.setText(backend.name)
else:
self.ui.nameEdit.setText('')
self.ui.backendInfo.setText(unicode(self.tr(
'
%s Backend %s
'
'Version: %s '
'Maintainer: %s '
'License: %s '
'%s'
'Description: %s '
'Capabilities: %s '))
% ('' if backend.icon_path else '',
backend.name.capitalize(),
backend.version,
backend.maintainer.replace('&', '&').replace('<', '<').replace('>', '>'),
backend.license,
(unicode(self.tr('Website: %s ')) % backend.website) if backend.website else '',
backend.description,
', '.join(sorted(cap.__name__.replace('ICap', '') for cap in backend.iter_caps()))))
if backend.has_caps(ICapAccount) and self.ui.nameEdit.isEnabled() and \
backend.klass.ACCOUNT_REGISTER_PROPERTIES is not None:
self.ui.registerButton.show()
else:
self.ui.registerButton.hide()
for key, field in backend.config.iteritems():
label = QLabel(u'%s:' % field.label)
qvalue = QtValue(field)
self.ui.configLayout.addRow(label, qvalue)
self.config_widgets[key] = (label, qvalue)
def proxyEditEnabled(self, state):
self.ui.proxyEdit.setEnabled(state)
def acceptBackend(self):
bname = unicode(self.ui.nameEdit.text())
selection = self.ui.backendsList.selectedItems()
if not selection:
QMessageBox.critical(self, self.tr('Unable to add a configured backend'),
self.tr('Please select a backend'))
return
try:
backend = self.weboob.modules_loader.get_or_load_module(unicode(selection[0].text()).lower())
except ModuleLoadError:
backend = None
if not backend:
QMessageBox.critical(self, self.tr('Unable to add a configured backend'),
self.tr('The selected backend does not exist.'))
return
params = {}
if not bname:
QMessageBox.critical(self, self.tr('Missing field'), self.tr('Please specify a backend name'))
return
if self.ui.nameEdit.isEnabled():
if not re.match(r'^[\w\-_]+$', bname):
QMessageBox.critical(self, self.tr('Invalid value'),
self.tr('The backend name can only contain letters and digits'))
return
if self.weboob.backends_config.backend_exists(bname):
QMessageBox.critical(self, self.tr('Unable to create backend'),
unicode(self.tr('Unable to create backend "%s": it already exists')) % bname)
return
if self.ui.proxyBox.isChecked():
params['_proxy'] = unicode(self.ui.proxyEdit.text())
if not params['_proxy']:
QMessageBox.critical(self, self.tr('Missing field'), self.tr('Please specify a proxy URL'))
return
config = backend.config.load(self.weboob, backend.name, bname, {}, nofail=True)
for key, field in config.iteritems():
label, qtvalue = self.config_widgets[key]
try:
value = qtvalue.get_value()
except ValueError, e:
QMessageBox.critical(self, self.tr('Invalid value'),
unicode(self.tr('Invalid value for field "%s":
%s')) % (field.label, e))
return
field.set(value.get())
try:
config.save(edit=not self.ui.nameEdit.isEnabled(), params=params)
except BackendAlreadyExists:
QMessageBox.critical(self, self.tr('Unable to create backend'),
unicode(self.tr('Unable to create backend "%s": it already exists')) % bname)
return
self.to_load.add(bname)
self.ui.configFrame.hide()
self.loadConfiguredBackendsList()
def rejectBackend(self):
self.ui.configFrame.hide()
def registerEvent(self):
selection = self.ui.backendsList.selectedItems()
if not selection:
return
try:
backend = self.weboob.modules_loader.get_or_load_module(unicode(selection[0].text()).lower())
except ModuleLoadError:
backend = None
if not backend:
return
dialog = QDialog(self)
vbox = QVBoxLayout(dialog)
if backend.website:
website = 'on the website %s' % backend.website
else:
website = 'with the backend %s' % backend.name
vbox.addWidget(QLabel('To create an account %s, please give these informations:' % website))
formlayout = QFormLayout()
props_widgets = OrderedDict()
for key, prop in backend.klass.ACCOUNT_REGISTER_PROPERTIES.iteritems():
widget = QtValue(prop)
formlayout.addRow(QLabel(u'%s:' % prop.label), widget)
props_widgets[prop.id] = widget
vbox.addLayout(formlayout)
buttonBox = QDialogButtonBox(dialog)
buttonBox.setStandardButtons(QDialogButtonBox.Ok|QDialogButtonBox.Cancel)
self.connect(buttonBox, SIGNAL("accepted()"), dialog.accept)
self.connect(buttonBox, SIGNAL("rejected()"), dialog.reject)
vbox.addWidget(buttonBox)
end = False
while not end:
end = True
if dialog.exec_():
account = Account()
account.properties = {}
for key, widget in props_widgets.iteritems():
try:
v = widget.get_value()
except ValueError, e:
QMessageBox.critical(self, self.tr('Invalid value'),
unicode(self.tr('Invalid value for field "%s":
%s')) % (key, e))
end = False
break
else:
account.properties[key] = v
if end:
try:
backend.klass.register_account(account)
except AccountRegisterError, e:
QMessageBox.critical(self, self.tr('Error during register'),
unicode(self.tr('Unable to register account %s:
%s')) % (website, e))
end = False
else:
for key, value in account.properties.iteritems():
if key in self.config_widgets:
self.config_widgets[key][1].set_value(value)
def run(self):
self.exec_()
ret = (len(self.to_load) > 0 or len(self.to_unload) > 0)
self.weboob.unload_backends(self.to_unload)
self.weboob.load_backends(names=self.to_load)
return ret
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/qt/backendcfg.ui 0000664 0000000 0000000 00000020763 11666415431 0030632 0 ustar 00root root 0000000 0000000
BackendCfg00622484Backends configurationQt::VerticalQAbstractItemView::NoEditTriggers4848falsefalsefalsetruetruefalsefalsefalsetruefalsetrueNameBackendQFrame::StyledPanelQFrame::RaisedAddRemoveQt::Vertical2040QFrame::StyledPanelQFrame::RaisedAvailable backends0032321true10QFrame::NoFrameQFrame::Plain00trueQFormLayout::ExpandingFieldsGrowName:Proxy:falseRegister an account...Qt::HorizontalQDialogButtonBox::Cancel|QDialogButtonBox::OkQDialogButtonBox::CloseconfiguredBackendsListaddButtonremoveButtonbackendsListbackendInfonameEditproxyBoxproxyEditregisterButtonconfigButtonBoxbuttonBoxbuttonBoxclicked(QAbstractButton*)BackendCfgaccept()312591312306
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/qt/qt.py 0000664 0000000 0000000 00000024564 11666415431 0027225 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import sys
import logging
import re
from threading import Event
from copy import copy
from PyQt4.QtCore import QTimer, SIGNAL, QObject, QString, QSize, QVariant, QMutex
from PyQt4.QtGui import QMainWindow, QApplication, QStyledItemDelegate, \
QStyleOptionViewItemV4, QTextDocument, QStyle, \
QAbstractTextDocumentLayout, QPalette, QMessageBox, \
QSpinBox, QLineEdit, QComboBox, QCheckBox, QInputDialog, \
QLineEdit
from weboob.core.ouiboube import Weboob
from weboob.core.scheduler import IScheduler
from weboob.tools.value import ValueInt, ValueBool, ValueBackendPassword
from ..base import BaseApplication
__all__ = ['QtApplication', 'QtMainWindow', 'QtDo', 'HTMLDelegate']
class QtScheduler(IScheduler):
def __init__(self, app):
self.app = app
self.count = 0
self.timers = {}
def schedule(self, interval, function, *args):
timer = QTimer()
timer.setInterval(interval * 1000)
timer.setSingleShot(True)
count = self.count
self.count += 1
timer.start()
self.app.connect(timer, SIGNAL("timeout()"), lambda: self.timeout(count, None, function, *args))
self.timers[count] = timer
def repeat(self, interval, function, *args):
timer = QTimer()
timer.setSingleShot(False)
count = self.count
self.count += 1
timer.start(0)
self.app.connect(timer, SIGNAL("timeout()"), lambda: self.timeout(count, interval, function, *args))
self.timers[count] = timer
def timeout(self, _id, interval, function, *args):
function(*args)
if interval is None:
self.timers.pop(_id)
else:
self.timers[_id].setInterval(interval * 1000)
def want_stop(self):
self.app.quit()
def run(self):
self.app.exec_()
class QCallbacksManager(QObject):
class Request(object):
def __init__(self):
self.event = Event()
self.answer = None
def __call__(self):
raise NotImplementedError()
class LoginRequest(Request):
def __init__(self, backend_name, value):
QCallbacksManager.Request.__init__(self)
self.backend_name = backend_name
self.value = value
def __call__(self):
password, ok = QInputDialog.getText(None,
'Password request',
'Please enter password for %s' % self.backend_name,
QLineEdit.Password)
return password
def __init__(self, weboob, parent=None):
QObject.__init__(self, parent)
self.weboob = weboob
self.weboob.callbacks['login'] = self.callback(self.LoginRequest)
self.mutex = QMutex()
self.requests = []
self.connect(self, SIGNAL('new_request'), self.do_request)
def callback(self, klass):
def cb(*args, **kwargs):
return self.add_request(klass(*args, **kwargs))
return cb
def do_request(self):
self.mutex.lock()
request = self.requests.pop()
request.answer = request()
request.event.set()
self.mutex.unlock()
def add_request(self, request):
self.mutex.lock()
self.requests.append(request)
self.mutex.unlock()
self.emit(SIGNAL('new_request'))
request.event.wait()
return request.answer
class QtApplication(QApplication, BaseApplication):
def __init__(self):
QApplication.__init__(self, sys.argv)
self.setApplicationName(self.APPNAME)
BaseApplication.__init__(self)
self.cbmanager = QCallbacksManager(self.weboob, self)
def create_weboob(self):
return Weboob(scheduler=QtScheduler(self))
class QtMainWindow(QMainWindow):
def __init__(self, parent=None):
QMainWindow.__init__(self, parent)
class QtDo(QObject):
def __init__(self, weboob, cb, eb=None):
QObject.__init__(self)
if not eb:
eb = self.default_eb
self.weboob = weboob
self.process = None
self.cb = cb
self.eb = eb
self.connect(self, SIGNAL('cb'), self.local_cb)
self.connect(self, SIGNAL('eb'), self.local_eb)
def do(self, *args, **kwargs):
self.process = self.weboob.do(*args, **kwargs)
self.process.callback_thread(self.thread_cb, self.thread_eb)
def default_eb(self, backend, error, backtrace):
# TODO display a messagebox
msg = u'%s' % error
if logging.root.level == logging.DEBUG:
msg += ' '
ul_opened = False
for line in backtrace.split('\n'):
m = re.match(' File (.*)', line)
if m:
if not ul_opened:
msg += '
'
ul_opened = True
else:
msg += '
'
msg += '
%s' % m.group(1)
else:
msg += ' %s' % line
if ul_opened:
msg += '
'
print >>sys.stderr, error
print >>sys.stderr, backtrace
QMessageBox.critical(None, self.tr('Error'), msg, QMessageBox.Ok)
def local_cb(self, backend, data):
self.cb(backend, data)
if not backend:
self.disconnect(self, SIGNAL('cb'), self.local_cb)
self.disconnect(self, SIGNAL('eb'), self.local_eb)
def local_eb(self, backend, error, backtrace):
self.eb(backend, error, backtrace)
self.disconnect(self, SIGNAL('cb'), self.local_cb)
self.disconnect(self, SIGNAL('eb'), self.local_eb)
def thread_cb(self, backend, data):
self.emit(SIGNAL('cb'), backend, data)
def thread_eb(self, backend, error, backtrace):
self.emit(SIGNAL('eb'), backend, error, backtrace)
class HTMLDelegate(QStyledItemDelegate):
def paint(self, painter, option, index):
optionV4 = QStyleOptionViewItemV4(option)
self.initStyleOption(optionV4, index)
style = optionV4.widget.style() if optionV4.widget else QApplication.style()
doc = QTextDocument()
doc.setHtml(optionV4.text)
# painting item without text
optionV4.text = QString()
style.drawControl(QStyle.CE_ItemViewItem, optionV4, painter)
ctx = QAbstractTextDocumentLayout.PaintContext()
# Hilight text if item is selected
if optionV4.state & QStyle.State_Selected:
ctx.palette.setColor(QPalette.Text, optionV4.palette.color(QPalette.Active, QPalette.HighlightedText))
textRect = style.subElementRect(QStyle.SE_ItemViewItemText, optionV4)
painter.save()
painter.translate(textRect.topLeft())
painter.setClipRect(textRect.translated(-textRect.topLeft()))
doc.documentLayout().draw(painter, ctx)
painter.restore()
def sizeHint(self, option, index):
optionV4 = QStyleOptionViewItemV4(option)
self.initStyleOption(optionV4, index)
doc = QTextDocument()
doc.setHtml(optionV4.text)
doc.setTextWidth(optionV4.rect.width())
return QSize(doc.idealWidth(), max(doc.size().height(), optionV4.decorationSize.height()))
class _QtValueStr(QLineEdit):
def __init__(self, value):
QLineEdit.__init__(self)
self._value = value
if value.default:
self.setText(unicode(value.default))
if value.masked:
self.setEchoMode(self.Password)
def set_value(self, value):
self._value = value
self.setText(self._value.get())
def get_value(self):
self._value.set(unicode(self.text()))
return self._value
class _QtValueBackendPassword(_QtValueStr):
def get_value(self):
self._value._domain = None
return _QtValueStr.get_value(self)
class _QtValueBool(QCheckBox):
def __init__(self, value):
QCheckBox.__init__(self)
self._value = value
if value.default:
self.setChecked(True)
def set_value(self, value):
self._value = value
self.setChecked(self._value.get())
def get_value(self):
self._value.set(self.isChecked())
return self._value
class _QtValueInt(QSpinBox):
def __init__(self, value):
QSpinBox.__init__(self)
self._value = value
if value.default:
self.setValue(int(value.default))
def set_value(self, value):
self._value = value
self.setValue(self._value.get())
def get_value(self):
self._value.set(self.getValue())
return self._value
class _QtValueChoices(QComboBox):
def __init__(self, value):
QComboBox.__init__(self)
self._value = value
for k, l in value.choices.iteritems():
self.addItem(l, QVariant(k))
if value.default == k:
self.setCurrentIndex(self.count()-1)
def set_value(self, value):
self._value = value
for i in xrange(self.count()):
if unicode(self.itemData(i).toString()) == self._value.get():
self.setCurrentIndex(i)
return
def get_value(self):
self._value.set(unicode(self.itemData(self.currentIndex()).toString()))
return self._value
def QtValue(value):
if isinstance(value, ValueBool):
klass = _QtValueBool
elif isinstance(value, ValueInt):
klass = _QtValueInt
elif isinstance(value, ValueBackendPassword):
klass = _QtValueBackendPassword
elif value.choices is not None:
klass = _QtValueChoices
else:
klass = _QtValueStr
return klass(copy(value))
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/repl.py 0000664 0000000 0000000 00000110344 11666415431 0027107 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz, Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import atexit
from cmd import Cmd
import logging
import locale
from optparse import OptionGroup, OptionParser, IndentedHelpFormatter
import os
import sys
from weboob.capabilities.base import FieldNotFound, CapBaseObject
from weboob.core import CallErrors
from weboob.core.modules import ModuleLoadError
from weboob.tools.application.formatters.iformatter import MandatoryFieldsNotFound
from weboob.tools.misc import to_unicode
from weboob.tools.path import Path
from weboob.tools.ordereddict import OrderedDict
from weboob.capabilities.collection import Collection, ICapCollection, CollectionNotFound
from .console import BackendNotGiven, ConsoleApplication
from .formatters.load import FormattersLoader, FormatterLoadError
from .results import ResultsCondition, ResultsConditionError
__all__ = ['ReplApplication', 'NotEnoughArguments']
class NotEnoughArguments(Exception):
pass
class ReplOptionParser(OptionParser):
def format_option_help(self, formatter=None):
if not formatter:
formatter = self.formatter
return '%s\n%s' % (formatter.format_commands(self.commands),
OptionParser.format_option_help(self, formatter))
class ReplOptionFormatter(IndentedHelpFormatter):
def format_commands(self, commands):
s = u''
for section, cmds in commands.iteritems():
if len(cmds) == 0:
continue
if len(s) > 0:
s += '\n'
s += '%s Commands:\n' % section
for c in cmds:
c = c.split('\n')[0]
s += ' %s\n' % c
return s
class ReplApplication(Cmd, ConsoleApplication):
"""
Base application class for Repl applications.
"""
SYNOPSIS = 'Usage: %prog [-dqv] [-b backends] [-cnfs] [command [arguments..]]\n'
SYNOPSIS += ' %prog [--help] [--version]'
DISABLE_REPL = False
EXTRA_FORMATTERS = {}
DEFAULT_FORMATTER = 'multiline'
COMMANDS_FORMATTERS = {}
weboob_commands = set(['backends', 'condition', 'count', 'formatter', 'inspect', 'logging', 'select', 'quit', 'ls', 'cd'])
hidden_commands = set(['EOF'])
def __init__(self):
Cmd.__init__(self)
# XXX can't use bold prompt because:
# 1. it causes problems when trying to get history (lines don't start
# at the right place).
# 2. when typing a line longer than term width, cursor goes at start
# of the same line instead of new line.
#self.prompt = self.BOLD + '%s> ' % self.APPNAME + self.NC
self.prompt = '%s> ' % self.APPNAME
self.intro = '\n'.join(('Welcome to %s%s%s v%s' % (self.BOLD, self.APPNAME, self.NC, self.VERSION),
'',
self.COPYRIGHT.encode(sys.stdout.encoding or locale.getpreferredencoding()),
'This program is free software: you can redistribute it and/or modify',
'it under the terms of the GNU Affero General Public License as published by',
'the Free Software Foundation, either version 3 of the License, or',
'(at your option) any later version.',
'',
'Type "help" to display available commands.',
'',
))
self.formatters_loader = FormattersLoader()
for key, klass in self.EXTRA_FORMATTERS.iteritems():
self.formatters_loader.register_formatter(key, klass)
self.formatter = None
self.commands_formatters = self.COMMANDS_FORMATTERS.copy()
ConsoleApplication.__init__(self, ReplOptionParser(self.SYNOPSIS, version=self._get_optparse_version()))
commands_help = self.get_commands_doc()
self._parser.commands = commands_help
self._parser.formatter = ReplOptionFormatter()
results_options = OptionGroup(self._parser, 'Results Options')
results_options.add_option('-c', '--condition', help='filter result items to display given a boolean expression')
results_options.add_option('-n', '--count', default='10', type='int',
help='get a maximum number of results (all backends merged)')
results_options.add_option('-s', '--select', help='select result item keys to display (comma separated)')
self._parser.add_option_group(results_options)
formatting_options = OptionGroup(self._parser, 'Formatting Options')
available_formatters = self.formatters_loader.get_available_formatters()
formatting_options.add_option('-f', '--formatter', choices=available_formatters,
help='select output formatter (%s)' % u', '.join(available_formatters))
formatting_options.add_option('--no-header', dest='no_header', action='store_true', help='do not display header')
formatting_options.add_option('--no-keys', dest='no_keys', action='store_true', help='do not display item keys')
formatting_options.add_option('-O', '--outfile', dest='outfile', help='file to export result')
self._parser.add_option_group(formatting_options)
self._interactive = False
self.objects = []
self.working_path = Path()
@property
def interactive(self):
return self._interactive
def _change_prompt(self):
path = self.working_path.tostring()
if len(path) > 0 and path != '/':
self.prompt = '%s:%s> ' % (self.APPNAME, path)
else:
self.prompt = '%s> ' % (self.APPNAME)
self.objects = []
def change_path(self, path):
self.working_path.fromstring(path)
self._change_prompt()
def add_object(self, obj):
self.objects.append(obj)
def _complete_object(self):
return ['%s@%s' % (obj.id, obj.backend) for obj in self.objects]
def parse_id(self, id, unique_backend=False):
if self.interactive:
try:
obj = self.objects[int(id) - 1]
except (IndexError,ValueError):
pass
else:
if isinstance(obj, CapBaseObject):
id = '%s@%s' % (obj.id, obj.backend)
try:
return ConsoleApplication.parse_id(self, id, unique_backend)
except BackendNotGiven, e:
backend_name = None
while not backend_name:
print 'This command works with an unique backend. Availables:'
for index, (name, backend) in enumerate(e.backends):
print '%s%d)%s %s%-15s%s %s' % (self.BOLD, index + 1, self.NC, self.BOLD, name, self.NC,
backend.DESCRIPTION)
i = self.ask('Select a backend to proceed with "%s"' % id)
if not i.isdigit():
if not i in dict(e.backends):
print >>sys.stderr, 'Error: %s is not a valid backend' % i
continue
backend_name = i
else:
i = int(i)
if i < 0 or i > len(e.backends):
print >>sys.stderr, 'Error: %s is not a valid choice' % i
continue
backend_name = e.backends[i-1][0]
return id, backend_name
def get_object(self, _id, method, fields=None):
if self.interactive:
try:
obj = self.objects[int(_id) - 1]
except (IndexError,ValueError):
pass
else:
if isinstance(obj, CapBaseObject):
for backend, obj in self.do('fillobj', obj, fields, backends=[obj.backend]):
if obj:
return obj
_id, backend_name = self.parse_id(_id)
backend_names = (backend_name,) if backend_name is not None else self.enabled_backends
for backend, obj in self.do(method, _id, backends=backend_names):
if obj:
backend.fillobj(obj, fields)
return obj
def unload_backends(self, *args, **kwargs):
self.objects = []
return ConsoleApplication.unload_backends(self, *args, **kwargs)
def load_backends(self, *args, **kwargs):
self.objects = []
return ConsoleApplication.load_backends(self, *args, **kwargs)
def main(self, argv):
cmd_args = argv[1:]
if cmd_args:
if cmd_args[0] == 'help':
self._parser.print_help()
self._parser.exit()
cmd_line = u' '.join(cmd_args)
cmds = cmd_line.split(';')
for cmd in cmds:
ret = self.onecmd(cmd)
if ret:
return ret
elif self.DISABLE_REPL:
self._parser.print_help()
self._parser.exit()
else:
try:
import readline
except ImportError:
pass
else:
history_filepath = os.path.join(self.weboob.WORKDIR, '%s_history' % self.APPNAME)
try:
readline.read_history_file(history_filepath)
except IOError:
pass
def savehist():
readline.write_history_file(history_filepath)
atexit.register(savehist)
self.intro += '\nLoaded backends: %s\n' % ', '.join(sorted(backend.name for backend in self.weboob.iter_backends()))
self._interactive = True
self.cmdloop()
def do(self, function, *args, **kwargs):
"""
Call Weboob.do(), passing count and selected fields given by user.
"""
backends = kwargs.pop('backends', None)
kwargs['backends'] = self.enabled_backends if backends is None else backends
kwargs['condition'] = self.condition
fields = self.selected_fields
if fields == '$direct':
fields = []
elif fields == '$full':
fields = None
return self.weboob.do(self._do_complete, self.options.count, fields, function, *args, **kwargs)
# -- command tools ------------
def parse_command_args(self, line, nb, req_n=None):
if line.strip() == '':
# because ''.split() = ['']
args = []
else:
args = line.strip().split(' ', nb - 1)
if req_n is not None and (len(args) < req_n):
raise NotEnoughArguments('Command needs %d arguments' % req_n)
if len(args) < nb:
args += tuple(None for i in xrange(nb - len(args)))
return args
# -- cmd.Cmd methods ---------
def postcmd(self, stop, line):
"""
This REPL method is overrided to return None instead of integers
to prevent stopping cmdloop().
"""
if not isinstance(stop, bool):
stop = None
return stop
def onecmd(self, line):
"""
This REPL method is overrided to catch some particular exceptions.
"""
line = to_unicode(line)
cmd, arg, ignored = self.parseline(line)
# Set the right formatter for the command.
try:
formatter_name = self.commands_formatters[cmd]
except KeyError:
formatter_name = self.DEFAULT_FORMATTER
self.set_formatter(formatter_name)
try:
try:
return super(ReplApplication, self).onecmd(line)
except CallErrors, e:
self.bcall_errors_handler(e)
except BackendNotGiven, e:
print >>sys.stderr, 'Error: %s' % str(e)
except NotEnoughArguments, e:
print >>sys.stderr, 'Error: not enough arguments. %s' % str(e)
except (KeyboardInterrupt, EOFError):
# ^C during a command process doesn't exit application.
print '\nAborted.'
finally:
self.flush()
def emptyline(self):
"""
By default, an emptyline repeats the previous command.
Overriding this function disables this behaviour.
"""
pass
def default(self, line):
print >>sys.stderr, 'Unknown command: "%s"' % line
return 2
def completenames(self, text, *ignored):
return [name for name in Cmd.completenames(self, text, *ignored) if name not in self.hidden_commands]
def path_completer(self, arg):
dirname = os.path.dirname(arg)
try:
childs = os.listdir(dirname or '.')
except OSError:
return ()
l = []
for child in childs:
path = os.path.join(dirname, child)
if os.path.isdir(path):
child += '/'
l.append(child)
return l
def complete(self, text, state):
"""
Override of the Cmd.complete() method to:
- add a space at end of proposals
- display only proposals for words which match the
text already written by user.
"""
super(ReplApplication, self).complete(text, state)
# When state = 0, Cmd.complete() set the 'completion_matches' attribute by
# calling the completion function. Then, for other states, it only try to
# get the right item in list.
# So that's the good place to rework the choices.
if state == 0:
self.completion_matches = [choice for choice in self.completion_matches if choice.startswith(text)]
try:
match = self.completion_matches[state]
except IndexError:
return None
else:
if match[-1] != '/':
return '%s ' % match
return match
# -- errors management -------------
def bcall_error_handler(self, backend, error, backtrace):
"""
Handler for an exception inside the CallErrors exception.
This method can be overrided to support more exceptions types.
"""
if isinstance(error, ResultsConditionError):
print >>sys.stderr, u'Error(%s): condition error: %s' % (backend.name, error)
else:
return super(ReplApplication, self).bcall_error_handler(backend, error, backtrace)
def bcall_errors_handler(self, errors):
if self.interactive:
ConsoleApplication.bcall_errors_handler(self, errors, 'Use "logging debug" option to print backtraces.')
else:
ConsoleApplication.bcall_errors_handler(self, errors)
# -- options related methods -------------
def _handle_options(self):
if self.options.formatter:
self.commands_formatters = {}
self.DEFAULT_FORMATTER = self.options.formatter
self.set_formatter(self.DEFAULT_FORMATTER)
if self.options.select:
self.selected_fields = self.options.select.split(',')
else:
self.selected_fields = '$direct'
if self.options.condition:
self.condition = ResultsCondition(self.options.condition)
else:
self.condition = None
if self.options.count == 0:
self._parser.error('Count must be at least 1, or negative for infinite')
elif self.options.count < 0:
# infinite search
self.options.count = None
return super(ReplApplication, self)._handle_options()
def get_command_help(self, command, short=False):
try:
doc = getattr(self, 'do_' + command).__doc__
except AttributeError:
return None
if doc:
doc = '\n'.join(line.strip() for line in doc.strip().split('\n'))
if not doc.startswith(command):
doc = '%s\n\n%s' % (command, doc)
if short:
doc = doc.split('\n')[0]
if not doc.startswith(command):
doc = command
return doc
def get_commands_doc(self):
names = set(name for name in self.get_names() if name.startswith('do_'))
appname = self.APPNAME.capitalize()
d = OrderedDict(((appname, []), ('Weboob', [])))
for name in sorted(names):
cmd = name[3:]
if cmd in self.hidden_commands.union(self.weboob_commands).union(['help']):
continue
elif getattr(self, name).__doc__:
d[appname].append(self.get_command_help(cmd))
else:
d[appname].append(cmd)
if not self.DISABLE_REPL:
for cmd in self.weboob_commands:
d['Weboob'].append(self.get_command_help(cmd))
return d
# -- default REPL commands ---------
def do_quit(self, arg):
"""
Quit the application.
"""
return True
def do_EOF(self, arg):
"""
Quit the command line interpreter when ^D is pressed.
"""
# print empty line for the next shell prompt to appear on the first column of the terminal
print
return self.do_quit(arg)
def do_help(self, arg=None):
if arg:
cmd_names = set(name[3:] for name in self.get_names() if name.startswith('do_'))
if arg in cmd_names:
command_help = self.get_command_help(arg)
if command_help is None:
logging.warning(u'Command "%s" is undocumented' % arg)
else:
self.stdout.write('%s\n' % command_help)
else:
print >>sys.stderr, 'Unknown command: "%s"' % arg
else:
cmds = self._parser.formatter.format_commands(self._parser.commands)
self.stdout.write('%s\n' % cmds)
self.stdout.write('Type "help " for more info about a command.\n')
return 2
def complete_backends(self, text, line, begidx, endidx):
choices = []
commands = ['enable', 'disable', 'only', 'list', 'add', 'register', 'edit', 'remove']
available_backends_names = set(backend.name for backend in self.weboob.iter_backends())
enabled_backends_names = set(backend.name for backend in self.enabled_backends)
args = line.split(' ')
if len(args) == 2:
choices = commands
elif len(args) >= 3:
if args[1] == 'enable':
choices = sorted(available_backends_names - enabled_backends_names)
elif args[1] == 'only':
choices = sorted(available_backends_names)
elif args[1] == 'disable':
choices = sorted(enabled_backends_names)
elif args[1] in ('add', 'register') and len(args) == 3:
self.weboob.modules_loader.load_all()
for name, module in sorted(self.weboob.modules_loader.loaded.iteritems()):
if not self.CAPS or self.caps_included(module.iter_caps(), self.CAPS.__name__):
choices.append(name)
elif args[1] == 'edit':
choices = sorted(available_backends_names)
elif args[1] == 'remove':
choices = sorted(available_backends_names)
return choices
def do_backends(self, line):
"""
backends [ACTION] [BACKEND_NAME]...
Select used backends.
ACTION is one of the following (default: list):
* enable enable given backends
* disable disable given backends
* only enable given backends and disable the others
* list display enabled and available backends
* add add a backend
* register register a new account on a website
* edit edit a backend
* remove remove a backend
"""
line = line.strip()
if line:
args = line.split()
else:
args = ['list']
action = args[0]
given_backend_names = args[1:]
for backend_name in given_backend_names:
if action in ('add', 'register'):
try:
module = self.weboob.modules_loader.get_or_load_module(backend_name)
except ModuleLoadError:
print >>sys.stderr, 'Backend "%s" does not exist.' % backend_name
return 1
else:
if self.CAPS and not self.caps_included(module.iter_caps(), self.CAPS.__name__):
print >>sys.stderr, 'Backend "%s" is not supported by this application => skipping.' % backend_name
return 1
else:
if backend_name not in [backend.name for backend in self.weboob.iter_backends()]:
print >>sys.stderr, 'Backend "%s" does not exist => skipping.' % backend_name
return 1
if action in ('enable', 'disable', 'only', 'add', 'register', 'edit', 'remove'):
if not given_backend_names:
print >>sys.stderr, 'Please give at least a backend name.'
return 2
given_backends = set(backend for backend in self.weboob.iter_backends() if backend.name in given_backend_names)
if action == 'enable':
for backend in given_backends:
self.enabled_backends.add(backend)
elif action == 'disable':
for backend in given_backends:
try:
self.enabled_backends.remove(backend)
except KeyError:
print >>sys.stderr, '%s is not enabled' % backend.name
elif action == 'only':
self.enabled_backends = set()
for backend in given_backends:
self.enabled_backends.add(backend)
elif action == 'list':
enabled_backends_names = set(backend.name for backend in self.enabled_backends)
disabled_backends_names = set(backend.name for backend in self.weboob.iter_backends()) - enabled_backends_names
print 'Enabled: %s' % ', '.join(enabled_backends_names)
if len(disabled_backends_names) > 0:
print 'Disabled: %s' % ', '.join(disabled_backends_names)
elif action == 'add':
for name in given_backend_names:
instname = self.add_backend(name)
if instname:
self.load_backends(names=[instname])
elif action == 'register':
for name in given_backend_names:
instname = self.register_backend(name)
if isinstance(instname, basestring):
self.load_backends(names=[instname])
elif action == 'edit':
for backend in given_backends:
enabled = backend in self.enabled_backends
self.unload_backends(names=[backend.name])
self.edit_backend(backend.name)
for newb in self.load_backends(names=[backend.name]).itervalues():
if not enabled:
self.enabled_backends.remove(newb)
elif action == 'remove':
for backend in given_backends:
self.weboob.backends_config.remove_backend(backend.name)
self.unload_backends(backend.name)
else:
print >>sys.stderr, 'Unknown action: "%s"' % action
return 1
if len(self.enabled_backends) == 0:
print >>sys.stderr, 'Warning: no more backends are loaded. %s is probably unusable.' % self.APPNAME.capitalize()
def complete_logging(self, text, line, begidx, endidx):
levels = ('debug', 'info', 'warning', 'error', 'quiet', 'default')
args = line.split(' ')
if len(args) == 2:
return levels
return ()
def do_logging(self, line):
"""
logging [LEVEL]
Set logging level.
Availables: debug, info, warning, error.
* quiet is an alias for error
* default is an alias for warning
"""
args = self.parse_command_args(line, 1, 0)
levels = (('debug', logging.DEBUG),
('info', logging.INFO),
('warning', logging.WARNING),
('error', logging.ERROR),
('quiet', logging.ERROR),
('default', logging.WARNING)
)
if not args[0]:
current = None
for label, level in levels:
if logging.root.level == level:
current = label
break
print 'Current level: %s' % current
return
levels = dict(levels)
try:
level = levels[args[0]]
except KeyError:
print >>sys.stderr, 'Level "%s" does not exist.' % args[0]
print >>sys.stderr, 'Availables: %s' % ' '.join(levels.iterkeys())
return 2
else:
logging.root.setLevel(level)
for handler in logging.root.handlers:
handler.setLevel(level)
def do_condition(self, line):
"""
condition [EXPRESSION | off]
If an argument is given, set the condition expression used to filter the results.
If the "off" value is given, conditional filtering is disabled.
If no argument is given, print the current condition expression.
"""
line = line.strip()
if line:
if line == 'off':
self.condition = None
else:
try:
self.condition = ResultsCondition(line)
except ResultsConditionError, e:
print >>sys.stderr, '%s' % e
return 2
else:
if self.condition is None:
print 'No condition is set.'
else:
print str(self.condition)
def do_count(self, line):
"""
count [NUMBER | off]
If an argument is given, set the maximum number of results fetched.
NUMBER must be at least 1.
"off" value disables counting, and allows infinite searches.
If no argument is given, print the current count value.
"""
line = line.strip()
if line:
if line == 'off':
self.options.count = None
else:
try:
count = int(line)
except ValueError:
print >>sys.stderr, 'Could not interpret "%s" as a number.' % line
return 2
else:
if count > 0:
self.options.count = count
else:
print >>sys.stderr, 'Number must be at least 1.'
return 2
else:
if self.options.count is None:
print 'Counting disabled.'
else:
print self.options.count
def complete_formatter(self, text, line, *ignored):
formatters = self.formatters_loader.get_available_formatters()
commands = ['list', 'option'] + formatters
options = ['header', 'keys']
option_values = ['on', 'off']
args = line.split(' ')
if len(args) == 2:
return commands
if args[1] == 'option':
if len(args) == 3:
return options
if len(args) == 4:
return option_values
elif args[1] in formatters:
return list(set(name[3:] for name in self.get_names() if name.startswith('do_')))
def do_formatter(self, line):
"""
formatter [list | FORMATTER [COMMAND] | option OPTION_NAME [on | off]]
If a FORMATTER is given, set the formatter to use.
You can add a COMMAND to apply the formatter change only to
a given command.
If the argument is "list", print the available formatters.
If the argument is "option", set the formatter options.
Valid options are: header, keys.
If on/off value is given, set the value of the option.
If not, print the current value for the option.
If no argument is given, print the current formatter.
"""
args = line.strip().split()
if args:
if args[0] == 'list':
print ', '.join(self.formatters_loader.get_available_formatters())
elif args[0] == 'option':
if len(args) > 1:
if len(args) == 2:
if args[1] == 'header':
print 'off' if self.options.no_header else 'on'
elif args[1] == 'keys':
print 'off' if self.options.no_keys else 'on'
else:
if args[2] not in ('on', 'off'):
print >>sys.stderr, 'Invalid value "%s". Please use "on" or "off" values.' % args[2]
return 2
else:
if args[1] == 'header':
self.options.no_header = True if args[2] == 'off' else False
elif args[1] == 'keys':
self.options.no_keys = True if args[2] == 'off' else False
else:
print >>sys.stderr, 'Don\'t know which option to set. Available options: header, keys.'
return 2
else:
if args[0] in self.formatters_loader.get_available_formatters():
if len(args) > 1:
self.commands_formatters[args[1]] = self.set_formatter(args[0])
else:
self.commands_formatters = {}
self.DEFAULT_FORMATTER = self.set_formatter(args[0])
else:
print >>sys.stderr, 'Formatter "%s" is not available.\n' \
'Available formatters: %s.' % (args[0], ', '.join(self.formatters_loader.get_available_formatters()))
return 1
else:
print 'Default formatter: %s' % self.DEFAULT_FORMATTER
for key, klass in self.commands_formatters.iteritems():
print 'Command "%s": %s' % (key, klass)
def do_select(self, line):
"""
select [FIELD_NAME]... | "$direct" | "$full"
If an argument is given, set the selected fields.
$direct selects all fields loaded in one http request.
$full selects all fields using as much http requests as necessary.
If no argument is given, print the currently selected fields.
"""
line = line.strip()
if line:
split = line.split()
if len(split) == 1 and split[0] in ('$direct', '$full'):
self.selected_fields = split[0]
else:
self.selected_fields = split
else:
if isinstance(self.selected_fields, basestring):
print self.selected_fields
else:
print ' '.join(self.selected_fields)
def complete_inspect(self, text, line, begidx, endidx):
return sorted(set(backend.name for backend in self.enabled_backends))
def do_inspect(self, line):
"""
inspect BACKEND_NAME
Display the HTML string of the current page of the specified backend's browser.
If webkit_mechanize_browser Python module is installed, HTML is displayed in a WebKit GUI.
"""
if len(self.enabled_backends) == 1:
backend = list(self.enabled_backends)[0]
else:
backend_name = line.strip()
if not backend_name:
print >>sys.stderr, 'Please specify a backend name.'
return 2
backends = set(backend for backend in self.enabled_backends if backend.name == backend_name)
if not backends:
print >>sys.stderr, 'No backend found for "%s"' % backend_name
return 1
backend = backends.pop()
if not backend.browser:
print >>sys.stderr, 'No browser created for backend "%s".' % backend.name
return 1
if not backend.browser.page:
print >>sys.stderr, 'The browser of %s is not on any page.' % backend.name
return 1
browser = backend.browser
data = browser.parser.tostring(browser.page.document)
try:
from webkit_mechanize_browser.browser import Browser
from weboob.tools.inspect import Page
except ImportError:
print data
else:
page = Page(core=browser, data=data, uri=browser._response.geturl())
browser = Browser(view=page.view)
def do_ls(self, line):
"""
ls
List objects in current path.
"""
self.objects = self._fetch_objects()
for obj in self.objects:
if isinstance(obj, CapBaseObject):
self.format(obj)
else:
print obj.title
self.flush()
def do_cd(self, line):
"""
cd PATH
Follow a path.
"""
line = line.encode('utf-8')
self.working_path.extend(line)
objects = self._fetch_objects()
if len(objects) == 0:
print >>sys.stderr, "Path: %s not found" % self.working_path.tostring()
self.working_path.restore()
return 1
self.objects = objects
self._change_prompt()
def _fetch_objects(self):
objects = []
path = self.working_path.get()
try:
for backend, res in self.do('iter_resources', path, caps=ICapCollection):
objects.append(res)
except CallErrors, errors:
for backend, error, backtrace in errors.errors:
if isinstance(error, CollectionNotFound):
pass
else:
self.bcall_error_handler(backend, error, backtrace)
return objects
def complete_cd(self, text, line, begidx, endidx):
directories = ['..']
mline = line.partition(' ')[2]
offs = len(mline) - len(text)
if len(self.objects) == 0:
self.objects = self._fetch_objects()
for obj in self.objects:
if isinstance(obj, Collection):
directories.append(obj.title)
return [s[offs:] for s in directories if s.startswith(mline)]
# -- formatting related methods -------------
def set_formatter(self, name):
"""
Set the current formatter from name.
It returns the name of the formatter which has been really set.
"""
try:
self.formatter = self.formatters_loader.build_formatter(name)
except FormatterLoadError, e:
print >>sys.stderr, '%s' % e
if self.DEFAULT_FORMATTER == name:
self.DEFAULT_FORMATTER = ReplApplication.DEFAULT_FORMATTER
print >>sys.stderr, 'Falling back to "%s".' % (self.DEFAULT_FORMATTER)
self.formatter = self.formatters_loader.build_formatter(self.DEFAULT_FORMATTER)
name = self.DEFAULT_FORMATTER
if self.options.no_header:
self.formatter.display_header = False
if self.options.no_keys:
self.formatter.display_keys = False
if self.options.outfile:
self.formatter.outfile = self.options.outfile
if self.interactive:
self.formatter.interactive = True
return name
def set_formatter_header(self, string):
self.formatter.set_header(string)
def format(self, result):
fields = self.selected_fields
if fields in ('$direct', '$full'):
fields = None
try:
self.formatter.format(obj=result, selected_fields=fields)
except FieldNotFound, e:
print >>sys.stderr, e
except MandatoryFieldsNotFound, e:
print >>sys.stderr, '%s Hint: select missing fields or use another formatter (ex: multiline).' % e
def flush(self):
self.formatter.flush()
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/application/results.py 0000664 0000000 0000000 00000005056 11666415431 0027651 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.core.bcall import IResultsCondition, ResultsConditionError
__all__ = ['ResultsCondition', 'ResultsConditionError']
class ResultsCondition(IResultsCondition):
condition_str = None
def __init__(self, condition_str):
condition_str = condition_str.replace(' OR ', ' or ') \
.replace(' AND ', ' and ') \
.replace(' NOT ', ' not ')
or_list = []
for _or in condition_str.split(' or '):
and_dict = {}
for _and in _or.split(' and '):
if '!=' in _and:
k, v = _and.split('!=')
k = k.strip() + '!'
elif '=' in _and:
k, v = _and.split('=')
else:
raise ResultsConditionError(u'Could not find = or != operator in sub-expression "%s"' % _and)
and_dict[k.strip()] = v.strip()
or_list.append(and_dict)
self.condition = or_list
self.condition_str = condition_str
def is_valid(self, obj):
d = dict(obj.iter_fields())
for _or in self.condition:
for k, v in _or.iteritems():
if k.endswith('!'):
k = k[:-1]
different = True
else:
different = False
if k in d:
if different:
if d[k] == v:
return False
else:
if d[k] != v:
return False
else:
raise ResultsConditionError(u'Field "%s" is not valid.' % k)
return True
def __str__(self):
return unicode(self).encode('utf-8')
def __unicode__(self):
return self.condition_str
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/backend.py 0000664 0000000 0000000 00000022543 11666415431 0025234 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import os
from threading import RLock
from copy import copy
from weboob.capabilities.base import CapBaseObject, FieldNotFound, IBaseCap, NotLoaded
from weboob.tools.misc import iter_fields
from weboob.tools.log import getLogger
from weboob.tools.value import ValuesDict
__all__ = ['BaseBackend', 'ObjectNotAvailable']
class ObjectNotAvailable(Exception):
pass
class BackendStorage(object):
def __init__(self, name, storage):
self.name = name
self.storage = storage
def set(self, *args):
if self.storage:
return self.storage.set('backends', self.name, *args)
def delete(self, *args):
if self.storage:
return self.storage.delete('backends', self.name, *args)
def get(self, *args, **kwargs):
if self.storage:
return self.storage.get('backends', self.name, *args, **kwargs)
else:
return kwargs.get('default', None)
def load(self, default):
if self.storage:
return self.storage.load('backends', self.name, default)
def save(self):
if self.storage:
return self.storage.save('backends', self.name)
class BackendConfig(ValuesDict):
modname = None
instname = None
weboob = None
def load(self, weboob, modname, instname, config, nofail=False):
"""
Load configuration from dict to create an instance.
@param weboob [Weboob] weboob object
@param modname [str] name of module
@param instname [str] name of instance of this backend
@param params [dict] parameters to load
@param nofail [bool] if true, this call can't fail.
@return [BackendConfig]
"""
cfg = BackendConfig()
cfg.modname = modname
cfg.instname = instname
cfg.weboob = weboob
for name, field in self.iteritems():
value = config.get(name, None)
if value is None:
if not nofail and field.required:
raise BaseBackend.ConfigError('Backend(%s): Configuration error: Missing parameter "%s" (%s)'
% (cfg.instname, name, field.description))
value = field.default
field = copy(field)
try:
field.load(cfg.instname, value, cfg.weboob.callbacks)
except ValueError, v:
if not nofail:
raise BaseBackend.ConfigError('Backend(%s): Configuration error for field "%s": %s' % (cfg.instname, name, v))
cfg[name] = field
return cfg
def dump(self):
settings = {}
for name, value in self.iteritems():
settings[name] = value.dump()
return settings
def save(self, edit=True, params=None):
assert self.modname is not None
assert self.instname is not None
assert self.weboob is not None
dump = self.dump()
if params is not None:
dump.update(params)
self.weboob.backends_config.add_backend(self.instname, self.modname, dump, edit)
class BaseBackend(object):
# Backend name.
NAME = None
# Name of the maintainer of this backend.
MAINTAINER = ''
# Email address of the maintainer.
EMAIL = ''
# Version of backend (for information only).
VERSION = ''
# Description
DESCRIPTION = ''
# License of this backend.
LICENSE = ''
# Configuration required for this backend.
# Values must be weboob.tools.value.Value objects.
CONFIG = BackendConfig()
# Storage
STORAGE = {}
# Browser class
BROWSER = None
# Supported objects to fill
# The key is the class and the value the method to call to fill
# Method prototype: method(object, fields)
# When the method is called, fields are only the one which are
# NOT yet filled.
OBJECTS = {}
class ConfigError(Exception): pass
def __enter__(self):
self.lock.acquire()
def __exit__(self, t, v, tb):
self.lock.release()
def __repr__(self):
return u"" % self.name
def __init__(self, weboob, name, config=None, storage=None, logger=None):
self.logger = getLogger(name, parent=logger)
self.weboob = weboob
self.name = name
self.lock = RLock()
# Private fields (which start with '_')
self._private_config = dict((key, value) for key, value in config.iteritems() if key.startswith('_'))
# Load configuration of backend.
self.config = self.CONFIG.load(weboob, self.NAME, self.name, config)
self.storage = BackendStorage(self.name, storage)
self.storage.load(self.STORAGE)
def deinit(self):
"""
This abstract method is called when the backend is unloaded.
"""
pass
class classprop(object):
def __init__(self, fget):
self.fget = fget
def __get__(self, inst, objtype=None):
if inst:
return self.fget(inst)
else:
return self.fget(objtype)
@classprop
def ICON(klass):
try:
import xdg.IconTheme
except ImportError:
getLogger(klass.NAME).debug(u'Python xdg module was not found. Please install it to read icon files.')
else:
return xdg.IconTheme.getIconPath(klass.NAME)
_browser = None
@property
def browser(self):
"""
Attribute 'browser'. The browser is created at the first call
of this attribute, to avoid useless pages access.
Note that the 'create_default_browser' method is called to create it.
"""
if self._browser is None:
self._browser = self.create_default_browser()
return self._browser
def create_default_browser(self):
"""
Method to overload to build the default browser in
attribute 'browser'.
"""
return self.create_browser()
def create_browser(self, *args, **kwargs):
"""
Build a browser from the BROWSER class attribute and the
given arguments.
"""
if not self.BROWSER:
return None
if '_proxy' in self._private_config:
kwargs['proxy'] = self._private_config['_proxy']
elif 'http_proxy' in os.environ:
kwargs['proxy'] = os.environ['http_proxy']
elif 'HTTP_PROXY' in os.environ:
kwargs['proxy'] = os.environ['HTTP_PROXY']
kwargs['logger'] = self.logger
return self.BROWSER(*args, **kwargs)
@classmethod
def iter_caps(klass):
def iter_caps(cls):
for base in cls.__bases__:
if issubclass(base, IBaseCap) and base != IBaseCap:
yield base
for cap in iter_caps(base):
yield cap
return iter_caps(klass)
def has_caps(self, *caps):
for c in caps:
if (isinstance(c, basestring) and c in [cap.__name__ for cap in self.iter_caps()]) or \
isinstance(self, c):
return True
return False
def fillobj(self, obj, fields=None):
"""
@param fields which fields to fill; if None, all fields are filled (list)
"""
def not_loaded(v):
return (v is NotLoaded or isinstance(v, CapBaseObject) and not v.__iscomplete__())
if isinstance(fields, basestring):
fields = (fields,)
missing_fields = []
if fields is None:
# Select all fields
if isinstance(obj, CapBaseObject):
fields = [item[0] for item in obj.iter_fields()]
else:
fields = [item[0] for item in iter_fields(obj)]
for field in fields:
if not hasattr(obj, field):
raise FieldNotFound(obj, field)
value = getattr(obj, field)
missing = False
if hasattr(value, '__iter__'):
for v in (value.itervalues() if isinstance(value, dict) else value):
if not_loaded(v):
missing = True
break
elif not_loaded(value):
missing = True
if missing:
missing_fields.append(field)
if not missing_fields:
return obj
assert type(obj) in self.OBJECTS, 'The object of type %s is not supported by the backend %s' % (type(obj), self)
for key, value in self.OBJECTS.iteritems():
if isinstance(obj, key):
self.logger.debug(u'Fill %r with fields: %s' % (obj, missing_fields))
return value(self, obj, missing_fields) or obj
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/browser/ 0000775 0000000 0000000 00000000000 11666415431 0024750 5 ustar 00root root 0000000 0000000 woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/browser/__init__.py 0000664 0000000 0000000 00000002503 11666415431 0027061 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.browser.browser import BrowserIncorrectPassword, BrowserBanned, \
BrowserUnavailable, BrowserRetry, \
BrowserHTTPNotFound, BrowserHTTPError, \
BasePage, BaseBrowser, BrokenPageError, \
StandardBrowser
__all__ = ['BrowserIncorrectPassword', 'BrowserBanned', 'BrowserUnavailable', 'BrowserRetry',
'BrowserHTTPNotFound', 'BrowserHTTPError', 'BasePage', 'BaseBrowser',
'BrokenPageError', 'StandardBrowser']
woob-037c35f40908f1833bda001fd8cdb2c9338a53d4-weboob/weboob/tools/browser/browser.py 0000664 0000000 0000000 00000045324 11666415431 0027015 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import with_statement
from copy import copy
from httplib import BadStatusLine
from logging import warning
import mechanize
import os
import sys
import re
import tempfile
from threading import RLock
import time
import urllib
import urllib2
import mimetypes
from weboob.tools.decorators import retry
from weboob.tools.log import getLogger
from weboob.tools.mech import ClientForm
ControlNotFoundError = ClientForm.ControlNotFoundError
from weboob.tools.parsers import get_parser
# Try to load cookies
try:
from .firefox_cookies import FirefoxCookieJar
except ImportError, e:
warning("Unable to store cookies: %s" % e)
HAVE_COOKIES = False
else:
HAVE_COOKIES = True
__all__ = ['BrowserIncorrectPassword', 'BrowserBanned', 'BrowserUnavailable', 'BrowserRetry',
'BrowserHTTPNotFound', 'BrowserHTTPError', 'BasePage', 'BaseBrowser', 'StandardBrowser']
# Exceptions
class BrowserIncorrectPassword(Exception):
pass
class BrowserBanned(BrowserIncorrectPassword):
pass
class BrowserUnavailable(Exception):
pass
class BrowserHTTPNotFound(BrowserUnavailable):
pass
class BrowserHTTPError(BrowserUnavailable):
pass
class BrowserRetry(Exception):
pass
class NoHistory(object):
"""
We don't want to fill memory with history
"""
def __init__(self):
pass
def add(self, request, response):
pass
def back(self, n, _response):
pass
def clear(self):
pass
def close(self):
pass
class BrokenPageError(Exception):
pass
class BasePage(object):
"""
Base page
"""
def __init__(self, browser, document, url='', groups=None, group_dict=None, logger=None):
self.browser = browser
self.parser = browser.parser
self.document = document
self.url = url
self.groups = groups
self.group_dict = group_dict
self.logger = getLogger('page', logger)
def on_loaded(self):
"""
Called when the page is loaded.
"""
pass
def check_location(func):
def inner(self, *args, **kwargs):
if args and isinstance(args[0], basestring):
url = args[0]
if url.startswith('/') and (not self.request or self.request.host != self.DOMAIN):
url = '%s://%s%s' % (self.PROTOCOL, self.DOMAIN, url)
url = re.sub('(.*)#.*', r'\1', url)
args = (url,) + args[1:]
return func(self, *args, **kwargs)
return inner
class StandardBrowser(mechanize.Browser):
# ------ Class attributes --------------------------------------
ENCODING = 'utf-8'
USER_AGENTS = {
'desktop_firefox': 'Mozilla/5.0 (X11; U; Linux x86_64; fr; rv:1.9.2.13) Gecko/20101209 Fedora/3.6.13-1.fc13 Firefox/3.6.13',
'android': 'Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17',
'microb': 'Mozilla/5.0 (X11; U; Linux armv7l; fr-FR; rv:1.9.2.3pre) Gecko/20100723 Firefox/3.5 Maemo Browser 1.7.4.8 RX-51 N900',
'wget': 'Wget/1.11.4',
}
USER_AGENT = USER_AGENTS['desktop_firefox']
SAVE_RESPONSES = False
DEBUG_HTTP = False
DEBUG_MECHANIZE = False
responses_dirname = None
responses_count = 0
# ------ Browser methods ---------------------------------------
# I'm not a robot, so disable the check of permissions in robots.txt.
default_features = copy(mechanize.Browser.default_features)
default_features.remove('_robots')
default_features.remove('_refresh')
def __init__(self, firefox_cookies=None, parser=None, history=NoHistory(), proxy=None, logger=None,
factory=None):
"""
Constructor of Browser.
@param filefox_cookies [str] Path to cookies' sqlite file.
@param parser [IParser] parser to use on HTML files.
@param history [object] History manager. Default value is an object
which does not keep history.
@param proxy [str] proxy URL to use.
@param factory [object] Mechanize factory. None to use Mechanize's default.
"""
mechanize.Browser.__init__(self, history=history, factory=factory)
self.logger = getLogger('browser', logger)
self.addheaders = [
['User-agent', self.USER_AGENT]
]
# Use a proxy
self.proxy = proxy
if proxy:
proto = 'http'
if proxy.find('://') >= 0:
proto, domain = proxy.split('://', 1)
else:
domain = proxy
self.set_proxies({proto: domain})
# Share cookies with firefox
if firefox_cookies and HAVE_COOKIES:
self._cookie = FirefoxCookieJar(self.DOMAIN, firefox_cookies)
self._cookie.load()
self.set_cookiejar(self._cookie)
else:
self._cookie = None
if parser is None:
parser = get_parser()()
elif isinstance(parser, (tuple,list,str,unicode)):
parser = get_parser(parser)()
self.parser = parser
self.lock = RLock()
if self.DEBUG_HTTP:
# display messages from httplib
self.set_debug_http(True)
if self.DEBUG_MECHANIZE:
# Enable log messages from mechanize.Browser
self.set_debug_redirects(True)
def __enter__(self):
self.lock.acquire()
def __exit__(self, t, v, tb):
self.lock.release()
@check_location
@retry(BrowserHTTPError, tries=3)
def openurl(self, *args, **kwargs):
"""
Open an URL but do not create a Page object.
"""
if_fail = kwargs.pop('if_fail', 'raise')
self.logger.debug('Opening URL "%s", %s' % (args, kwargs))
try:
return mechanize.Browser.open_novisit(self, *args, **kwargs)
except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e:
if if_fail == 'raise':
raise self.get_exception(e)('%s (url="%s")' % (e, args and args[0] or 'None'))
else:
return None
except (mechanize.BrowserStateError, BrowserRetry):
self.home()
return mechanize.Browser.open(self, *args, **kwargs)
def get_exception(self, e):
if isinstance(e, urllib2.HTTPError) and hasattr(e, 'getcode') and e.getcode() == 404:
return BrowserHTTPNotFound
else:
return BrowserHTTPError
def readurl(self, url, *args, **kwargs):
"""
Download URL data specifying what to do on failure (nothing by default).
"""
if not 'if_fail' in kwargs:
kwargs['if_fail'] = None
result = self.openurl(url, *args, **kwargs)
if result:
if self.SAVE_RESPONSES:
self.save_response(result)
return result.read()
else:
return None
def save_response(self, result, warning=False):
"""
Save a stream to a temporary file, and log its name.
The stream is rewinded after saving.
"""
if self.responses_dirname is None:
self.responses_dirname = tempfile.mkdtemp(prefix='weboob_session_')
print >>sys.stderr, 'Debug data will be saved in this directory: %s' % self.responses_dirname
# get the content-type, remove optionnal charset part
mimetype = result.info().get('Content-Type', '').split(';')[0]
# due to http://bugs.python.org/issue1043134
if mimetype == 'text/plain':
ext = '.txt'
else:
# try to get an extension (and avoid adding 'None')
ext = mimetypes.guess_extension(mimetype, False) or ''
response_filepath = os.path.join(self.responses_dirname, unicode(self.responses_count)+ext)
with open(response_filepath, 'w') as f:
f.write(result.read())
result.seek(0)
match_filepath = os.path.join(self.responses_dirname, 'url_response_match.txt')
with open(match_filepath, 'a') as f:
f.write('%s\t%s\n' % (result.geturl(), os.path.basename(response_filepath)))
self.responses_count += 1
msg = u'Response saved to %s' % response_filepath
if warning:
self.logger.warning(msg)
else:
self.logger.info(msg)
def get_document(self, result):
return self.parser.parse(result, self.ENCODING)
def location(self, *args, **kwargs):
return self.get_document(self.openurl(*args, **kwargs))
@staticmethod
def buildurl(base, *args, **kwargs):
"""
Build an URL and escape arguments.
You can give a serie of tuples in *args (and the order is keept), or
a dict in **kwargs (but the order is lost).
Example:
>>> buildurl('/blah.php', ('a', '&'), ('b', '=')
'/blah.php?a=%26&b=%3D'
>>> buildurl('/blah.php', a='&', 'b'='=')
'/blah.php?b=%3D&a=%26'
"""
if not args:
args = kwargs
if not args:
return base
else:
return '%s?%s' % (base, urllib.urlencode(args))
def str(self, s):
if isinstance(s, unicode):
s = s.encode('iso-8859-15', 'replace')
return s
def set_field(self, args, label, field=None, value=None, is_list=False):
"""
Set a value to a form field.
@param args [dict] arguments where to look for value.
@param label [str] label in args.
@param field [str] field name. If None, use label instead.
@param value [str] value to give on field.
@param is_list [bool] the field is a list.
"""
try:
if not field:
field = label
if args.get(label, None) is not None:
if not value:
if is_list:
if isinstance(is_list, (list, tuple)):
try:
value = [self.str(is_list.index(args[label]))]
except ValueError, e:
if args[label]:
print >>sys.stderr, '[%s] %s: %s' % (label, args[label], e)
return
else:
value = [self.str(args[label])]
else:
value = self.str(args[label])
self[field] = value
except ControlNotFoundError:
return
class BaseBrowser(StandardBrowser):
"""
Base browser class to navigate on a website.
"""
# ------ Class attributes --------------------------------------
DOMAIN = None
PROTOCOL = 'http'
PAGES = {}
# ------ Abstract methods --------------------------------------
def home(self):
"""
Go to the home page.
"""
if self.DOMAIN is not None:
self.location('%s://%s/' % (self.PROTOCOL, self.DOMAIN))
def login(self):
"""
Login to the website.
This function is called when is_logged() returns False and the password
attribute is not None.
"""
raise NotImplementedError()
def is_logged(self):
"""
Return True if we are logged on website. When Browser tries to access
to a page, if this method returns False, it calls login().
It is never called if the password attribute is None.
"""
raise NotImplementedError()
# ------ Browser methods ---------------------------------------
def __init__(self, username=None, password=None, firefox_cookies=None,
parser=None, history=NoHistory(), proxy=None, logger=None,
factory=None, get_home=True):
"""
Constructor of Browser.
@param username [str] username on website.
@param password [str] password on website. If it is None, Browser will
not try to login.
@param filefox_cookies [str] Path to cookies' sqlite file.
@param parser [IParser] parser to use on HTML files.
@param hisory [object] History manager. Default value is an object
which does not keep history.
@param proxy [str] proxy URL to use.
@param factory [object] Mechanize factory. None to use Mechanize's default.
@param get_home [bool] Try to get the homepage.
"""
StandardBrowser.__init__(self, firefox_cookies, parser, history, proxy, logger, factory)
self.page = None
self.last_update = 0.0
self.username = username
self.password = password
if self.password and get_home:
try:
self.home()
# Do not abort the build of browser when the website is down.
except BrowserUnavailable:
pass
def pageaccess(func):
"""
Decorator to use around a method which access to a page.
"""
def inner(self, *args, **kwargs):
if not self.page or self.password and not self.page.is_logged():
self.home()
return func(self, *args, **kwargs)
return inner
@pageaccess
def keepalive(self):
self.home()
def submit(self, *args, **kwargs):
"""
Submit the selected form.
"""
try:
self._change_location(mechanize.Browser.submit(self, *args, **kwargs))
except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e:
self.page = None
raise self.get_exception(e)(e)
except (mechanize.BrowserStateError, BrowserRetry), e:
raise BrowserUnavailable(e)
def is_on_page(self, pageCls):
return isinstance(self.page, pageCls)
def absurl(self, rel):
if rel is None:
return None
if not rel.startswith('/'):
rel = '/' + rel
return '%s://%s%s' % (self.PROTOCOL, self.DOMAIN, rel)
def follow_link(self, *args, **kwargs):
try:
self._change_location(mechanize.Browser.follow_link(self, *args, **kwargs))
except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e:
self.page = None
raise self.get_exception(e)('%s (url="%s")' % (e, args and args[0] or 'None'))
except (mechanize.BrowserStateError, BrowserRetry), e:
self.home()
raise BrowserUnavailable(e)
@check_location
@retry(BrowserHTTPError, tries=3)
def location(self, *args, **kwargs):
"""
Change location of browser on an URL.
When the page is loaded, it looks up PAGES to find a regexp which
matches, and create the object. Then, the 'on_loaded' method of
this object is called.
If a password is set, and is_logged() returns False, it tries to login
with login() and reload the page.
"""
keep_args = copy(args)
keep_kwargs = kwargs.copy()
no_login = kwargs.pop('no_login', False)
try:
self._change_location(mechanize.Browser.open(self, *args, **kwargs), no_login=no_login)
except BrowserRetry:
if not self.page or not args or self.page.url != args[0]:
keep_kwargs['no_login'] = True
self.location(*keep_args, **keep_kwargs)
except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e:
self.page = None
raise self.get_exception(e)('%s (url="%s")' % (e, args and args[0] or 'None'))
except mechanize.BrowserStateError:
self.home()
self.location(*keep_args, **keep_kwargs)
# DO NOT ENABLE THIS FUCKING PEACE OF CODE EVEN IF IT WOULD BE BETTER
# TO SANITARIZE FUCKING HTML.
#def _set_response(self, response, *args, **kwargs):
# import time
# if response and hasattr(response, 'set_data'):
# print time.time()
# r = response.read()
# start = 0
# end = 0
# new = ''
# lowr = r.lower()
# start = lowr[end:].find('')
# new += r[start_stop:end].replace('<', '<').replace('>', '>')
# start = end + lowr[end:].find('