Commit 38a550d2 authored by Bezleputh's avatar Bezleputh Committed by Romain Bignon

[feedly] New module feedly

parent 4a22e687
# -*- coding: utf-8 -*-
# Copyright(C) 2014 Bezleputh
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .backend import FeedlyBackend
__all__ = ['FeedlyBackend']
# -*- coding: utf-8 -*-
# Copyright(C) 2014 Bezleputh
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.capabilities.collection import ICapCollection
from weboob.capabilities.messages import ICapMessages, Message, Thread
from weboob.tools.value import Value, ValueBackendPassword
from .browser import FeedlyBrowser
from .google import GoogleBrowser
__all__ = ['FeedlyBackend']
class FeedlyBackend(BaseBackend, ICapMessages, ICapCollection):
NAME = 'feedly'
DESCRIPTION = u'handle the popular RSS reading service Feedly'
MAINTAINER = u'Bezleputh'
EMAIL = 'carton_ben@yahoo.fr'
LICENSE = 'AGPLv3+'
VERSION = '0.j'
STORAGE = {'seen': []}
CONFIG = BackendConfig(Value('username', label='Username', default=''),
ValueBackendPassword('password', label='Password', default=''))
BROWSER = FeedlyBrowser
def iter_resources(self, objs, split_path):
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
return self.browser.get_categories()
if collection.path_level == 1:
return self.browser.get_feeds(split_path[0])
if collection.path_level == 2:
url = self.browser.get_feed_url(split_path[0], split_path[1])
threads = []
for article in self.browser.get_unread_feed(url):
thread = self.get_thread(article.id, article)
threads.append(thread)
return threads
def validate_collection(self, objs, collection):
if collection.path_level in [0, 1, 2]:
return
def get_thread(self, id, entry=None):
if isinstance(id, Thread):
thread = id
id = thread.id
else:
thread = Thread(id)
if entry is None:
url = id.split('#')[0]
for article in self.browser.get_unread_feed(url):
if article.id == id:
entry = article
if entry is None:
return None
if not thread.id in self.storage.get('seen', default=[]):
entry.flags = Message.IS_UNREAD
entry.thread = thread
thread.title = entry.title
thread.root = entry
return thread
def iter_unread_messages(self):
for thread in self.iter_threads():
for m in thread.iter_all_messages():
if m.flags & m.IS_UNREAD:
yield m
def iter_threads(self):
for article in self.browser.iter_threads():
yield self.get_thread(article.id, article)
def set_message_read(self, message):
self.browser.set_message_read(message.thread.id.split('#')[-1])
self.storage.get('seen', default=[]).append(message.thread.id)
self.storage.save()
def fill_thread(self, thread, fields):
return self.get_thread(thread)
def create_default_browser(self):
username = self.config['username'].get()
if username:
password = self.config['password'].get()
login_browser = GoogleBrowser(username, password,
'https://feedly.com/v3/auth/callback&scope=profile+email&state=Ak7fo397ImkiOiJmZWVkbHkiLCJyIjoiaHR0cDovL2ZlZWRseS5jb20vZmVlZGx5Lmh0bWwiLCJwIjoiR29vZ)2xlUGx1cyIsImMiOiJmZWVkbHkuZGVza3RvcCAyMC40Ljc3NSJ9')
else:
password = None
login_browser = None
return self.create_browser(username, password, login_browser)
OBJECTS = {Thread: fill_thread}
# -*- coding: utf-8 -*-
# Copyright(C) 2014 Bezleputh
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import simplejson
from weboob.capabilities.collection import Collection
from weboob.tools.browser2 import LoginBrowser, URL, need_login
from .pages import EssentialsPage, TokenPage, ContentsPage, PreferencesPage
__all__ = ['FeedlyBrowser']
class FeedlyBrowser(LoginBrowser):
BASEURL = 'http://www.feedly.com'
essentials = URL('http://s3.feedly.com/essentials/essentials_fr.json', EssentialsPage)
token = URL('v3/auth/token', TokenPage)
contents = URL('v3/streams/contents', ContentsPage)
preferences = URL('v3/preferences', PreferencesPage)
marker = URL('v3/markers')
def __init__(self, username, password, login_browser, *args, **kwargs):
super(FeedlyBrowser, self).__init__(username, password, *args, **kwargs)
self.login_browser = login_browser
self.user_id = None
def do_login(self):
if self.login_browser.code is None or self.user_id is None:
self.login_browser.do_login()
params = {'code': self.login_browser.code,
'client_id': 'feedly',
'client_secret': '0XP4XQ07VVMDWBKUHTJM4WUQ',
'redirect_uri': 'http://dev.feedly.com/feedly.html',
'grant_type': 'authorization_code'}
token, self.user_id = self.token.go(data=params).get_token()
self.session.headers['X-Feedly-Access-Token'] = token
@need_login
def iter_threads(self):
params = {'streamId': 'user/%s/category/global.all' % self.user_id,
'unreadOnly': 'true',
'ranked': 'newest',
'count': '100'}
return self.contents.go(params=params).get_articles()
def get_unread_feed(self, url):
params = {'streamId': url,
'backfill': 'true',
'boostMustRead': 'true',
'unreadOnly': 'true'}
return self.contents.go(params=params).get_articles()
def get_categories(self):
if self.username is not None and self.password is not None:
return self.get_logged_categories()
return self.essentials.go().get_categories()
@need_login
def get_logged_categories(self):
user_categories = list(self.preferences.go().get_categories())
user_categories.append(Collection(['global.saved'], 'Saved'))
return user_categories
def get_feeds(self, category):
if self.username is not None and self.password is not None:
return self.get_logged_feeds(category)
return self.essentials.go().get_feeds(category)
@need_login
def get_logged_feeds(self, category):
if category == 'global.saved':
type = 'tag'
else:
type = 'category'
url = 'user/%s/%s/%s' % (self.user_id, type, category)
return self.get_unread_feed(url)
def get_feed_url(self, category, feed):
return self.essentials.go().get_feed_url(category, feed)
@need_login
def set_message_read(self, _id):
datas = {'action': 'markAsRead',
'type': 'entries',
'entryIds': [_id]}
self.marker.open(data=simplejson.dumps(datas))
# -*- coding: utf-8 -*-
# Copyright(C) 2014 Bezleputh
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from urlparse import urlparse, parse_qs
from weboob.tools.browser2 import LoginBrowser, URL, HTMLPage
from weboob.tools.exceptions import BrowserIncorrectPassword
__all__ = ['GoogleBrowser', 'GoogleLoginPage']
class GoogleLoginPage(HTMLPage):
def login(self, login, passwd):
form = self.get_form('//form[@id="gaia_loginform"]')
form['Email'] = login
form['Passwd'] = passwd
form.submit()
class GoogleBrowser(LoginBrowser):
BASEURL = 'https://accounts.google.com'
code = None
google_login = URL('https://accounts.google.com/(?P<auth>.+)', GoogleLoginPage)
def __init__(self, username, password, redirect_uri, *args, **kwargs):
super(GoogleBrowser, self).__init__(username, password, *args, **kwargs)
self.redirect_uri = redirect_uri
def do_login(self):
params = {'response_type': 'code',
'client_id': '534890559860-r6gn7e3agcpiriehe63dkeus0tpl5i4i.apps.googleusercontent.com',
'redirect_uri': self.redirect_uri}
queryString = "&".join([key+'='+value for key, value in params.items()])
self.google_login.go(auth='o/oauth2/auth', params=queryString).login(self.username, self.password)
try:
self.code = parse_qs(urlparse(self.url).query).get('code')[0]
except:
raise BrowserIncorrectPassword()
# -*- coding: utf-8 -*-
# Copyright(C) 2014 Bezleputh
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from datetime import datetime
from weboob.capabilities.messages import Message
from weboob.capabilities.collection import Collection
from weboob.tools.browser2.page import JsonPage, ListElement, method, ItemElement
from weboob.tools.browser2.filters import CleanText, Dict, Format, CleanHTML
__all__ = ['TokenPage', 'ContentsPage', 'PreferencesPage']
class DictElement(ListElement):
def find_elements(self):
if self.item_xpath is not None:
for el in self.el.get(self.item_xpath):
yield el
else:
yield self.el
class ContentsPage(JsonPage):
@method
class get_articles(DictElement):
item_xpath = 'items'
class item(ItemElement):
klass = Message
obj_id = Format(u'%s#%s', CleanText(Dict('origin/streamId')), CleanText(Dict('id')))
obj_sender = CleanText(Dict('author', default=u''))
obj_title = Format(u'%s - %s', CleanText(Dict('origin/title', default=u'')), CleanText(Dict('title')))
def obj_date(self):
return datetime.fromtimestamp(Dict('published')(self.el) / 1e3)
def obj_content(self):
if 'content' in self.el.keys():
return Format(u'%s%s\r\n',
CleanHTML(Dict('content/content')), CleanText(Dict('origin/htmlUrl')))(self.el)
elif 'summary' in self.el.keys():
return Format(u'%s%s\r\n',
CleanHTML(Dict('summary/content')), CleanText(Dict('origin/htmlUrl')))(self.el)
else:
return ''
class TokenPage(JsonPage):
def get_token(self):
return self.doc['access_token'], self.doc['id']
class EssentialsPage(JsonPage):
def get_categories(self):
for category in self.doc:
name = u'%s' % category.get('label')
yield Collection([name], name)
def get_feeds(self, label):
for category in self.doc:
if category.get('label') == label:
feeds = category.get('subscriptions')
for feed in feeds:
yield Collection([label, feed.get('title')])
def get_feed_url(self, _category, _feed):
for category in self.doc:
if category.get('label') == _category:
feeds = category.get('subscriptions')
for feed in feeds:
if feed.get('title') == _feed:
return feed.get('id')
class PreferencesPage(JsonPage):
def get_categories(self):
for category, value in self.doc.items():
if value in [u"shown", u"hidden"]:
yield Collection([category])
# -*- coding: utf-8 -*-
# Copyright(C) 2014 Bezleputh
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from nose.plugins.skip import SkipTest
from weboob.tools.test import BackendTest
class FeedlyTest(BackendTest):
BACKEND = 'feedly'
def test_feedly(self):
if self.backend.browser.username:
l1 = list(self.backend.iter_threads())
assert len(l1)
thread = self.backend.get_thread(l1[0].id)
assert len(thread.root.content)
else:
raise SkipTest("User credentials not defined")
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment