diff --git a/modules/feedly/__init__.py b/modules/feedly/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a3c4a9dd123a68c640c9ccbfb6418124997f96d1 --- /dev/null +++ b/modules/feedly/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Bezleputh +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from .backend import FeedlyBackend + + +__all__ = ['FeedlyBackend'] diff --git a/modules/feedly/backend.py b/modules/feedly/backend.py new file mode 100644 index 0000000000000000000000000000000000000000..b898185985e77c82515180374037c9366fafa08e --- /dev/null +++ b/modules/feedly/backend.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Bezleputh +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.backend import BaseBackend, BackendConfig +from weboob.capabilities.collection import ICapCollection +from weboob.capabilities.messages import ICapMessages, Message, Thread +from weboob.tools.value import Value, ValueBackendPassword + +from .browser import FeedlyBrowser +from .google import GoogleBrowser + +__all__ = ['FeedlyBackend'] + + +class FeedlyBackend(BaseBackend, ICapMessages, ICapCollection): + NAME = 'feedly' + DESCRIPTION = u'handle the popular RSS reading service Feedly' + MAINTAINER = u'Bezleputh' + EMAIL = 'carton_ben@yahoo.fr' + LICENSE = 'AGPLv3+' + VERSION = '0.j' + STORAGE = {'seen': []} + CONFIG = BackendConfig(Value('username', label='Username', default=''), + ValueBackendPassword('password', label='Password', default='')) + + BROWSER = FeedlyBrowser + + def iter_resources(self, objs, split_path): + collection = self.get_collection(objs, split_path) + if collection.path_level == 0: + return self.browser.get_categories() + + if collection.path_level == 1: + return self.browser.get_feeds(split_path[0]) + + if collection.path_level == 2: + url = self.browser.get_feed_url(split_path[0], split_path[1]) + threads = [] + for article in self.browser.get_unread_feed(url): + thread = self.get_thread(article.id, article) + threads.append(thread) + return threads + + def validate_collection(self, objs, collection): + if collection.path_level in [0, 1, 2]: + return + + def get_thread(self, id, entry=None): + if isinstance(id, Thread): + thread = id + id = thread.id + else: + thread = Thread(id) + if entry is None: + url = id.split('#')[0] + for article in self.browser.get_unread_feed(url): + if article.id == id: + entry = article + if entry is None: + return None + + if not thread.id in self.storage.get('seen', default=[]): + entry.flags = Message.IS_UNREAD + + entry.thread = thread + thread.title = entry.title + thread.root = entry + return thread + + def iter_unread_messages(self): + for thread in self.iter_threads(): + for m in thread.iter_all_messages(): + if m.flags & m.IS_UNREAD: + yield m + + def iter_threads(self): + for article in self.browser.iter_threads(): + yield self.get_thread(article.id, article) + + def set_message_read(self, message): + self.browser.set_message_read(message.thread.id.split('#')[-1]) + self.storage.get('seen', default=[]).append(message.thread.id) + self.storage.save() + + def fill_thread(self, thread, fields): + return self.get_thread(thread) + + def create_default_browser(self): + username = self.config['username'].get() + if username: + password = self.config['password'].get() + login_browser = GoogleBrowser(username, password, + 'https://feedly.com/v3/auth/callback&scope=profile+email&state=Ak7fo397ImkiOiJmZWVkbHkiLCJyIjoiaHR0cDovL2ZlZWRseS5jb20vZmVlZGx5Lmh0bWwiLCJwIjoiR29vZ)2xlUGx1cyIsImMiOiJmZWVkbHkuZGVza3RvcCAyMC40Ljc3NSJ9') + else: + password = None + login_browser = None + return self.create_browser(username, password, login_browser) + + OBJECTS = {Thread: fill_thread} diff --git a/modules/feedly/browser.py b/modules/feedly/browser.py new file mode 100644 index 0000000000000000000000000000000000000000..2ff0a37de2d8d3ddb71453df9270655e5a27291d --- /dev/null +++ b/modules/feedly/browser.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Bezleputh +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +import simplejson + +from weboob.capabilities.collection import Collection +from weboob.tools.browser2 import LoginBrowser, URL, need_login +from .pages import EssentialsPage, TokenPage, ContentsPage, PreferencesPage + + +__all__ = ['FeedlyBrowser'] + + +class FeedlyBrowser(LoginBrowser): + BASEURL = 'http://www.feedly.com' + + essentials = URL('http://s3.feedly.com/essentials/essentials_fr.json', EssentialsPage) + token = URL('v3/auth/token', TokenPage) + contents = URL('v3/streams/contents', ContentsPage) + preferences = URL('v3/preferences', PreferencesPage) + marker = URL('v3/markers') + + def __init__(self, username, password, login_browser, *args, **kwargs): + super(FeedlyBrowser, self).__init__(username, password, *args, **kwargs) + self.login_browser = login_browser + self.user_id = None + + def do_login(self): + if self.login_browser.code is None or self.user_id is None: + self.login_browser.do_login() + params = {'code': self.login_browser.code, + 'client_id': 'feedly', + 'client_secret': '0XP4XQ07VVMDWBKUHTJM4WUQ', + 'redirect_uri': 'http://dev.feedly.com/feedly.html', + 'grant_type': 'authorization_code'} + + token, self.user_id = self.token.go(data=params).get_token() + self.session.headers['X-Feedly-Access-Token'] = token + + @need_login + def iter_threads(self): + params = {'streamId': 'user/%s/category/global.all' % self.user_id, + 'unreadOnly': 'true', + 'ranked': 'newest', + 'count': '100'} + return self.contents.go(params=params).get_articles() + + def get_unread_feed(self, url): + params = {'streamId': url, + 'backfill': 'true', + 'boostMustRead': 'true', + 'unreadOnly': 'true'} + return self.contents.go(params=params).get_articles() + + def get_categories(self): + if self.username is not None and self.password is not None: + return self.get_logged_categories() + return self.essentials.go().get_categories() + + @need_login + def get_logged_categories(self): + user_categories = list(self.preferences.go().get_categories()) + user_categories.append(Collection(['global.saved'], 'Saved')) + return user_categories + + def get_feeds(self, category): + if self.username is not None and self.password is not None: + return self.get_logged_feeds(category) + return self.essentials.go().get_feeds(category) + + @need_login + def get_logged_feeds(self, category): + if category == 'global.saved': + type = 'tag' + else: + type = 'category' + url = 'user/%s/%s/%s' % (self.user_id, type, category) + return self.get_unread_feed(url) + + def get_feed_url(self, category, feed): + return self.essentials.go().get_feed_url(category, feed) + + @need_login + def set_message_read(self, _id): + datas = {'action': 'markAsRead', + 'type': 'entries', + 'entryIds': [_id]} + self.marker.open(data=simplejson.dumps(datas)) diff --git a/modules/feedly/google.py b/modules/feedly/google.py new file mode 100644 index 0000000000000000000000000000000000000000..aacb8a15a52244cd9abf2cf6e455a3479bd9bf2d --- /dev/null +++ b/modules/feedly/google.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Bezleputh +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from urlparse import urlparse, parse_qs + +from weboob.tools.browser2 import LoginBrowser, URL, HTMLPage +from weboob.tools.exceptions import BrowserIncorrectPassword + +__all__ = ['GoogleBrowser', 'GoogleLoginPage'] + + +class GoogleLoginPage(HTMLPage): + def login(self, login, passwd): + form = self.get_form('//form[@id="gaia_loginform"]') + form['Email'] = login + form['Passwd'] = passwd + form.submit() + + +class GoogleBrowser(LoginBrowser): + BASEURL = 'https://accounts.google.com' + + code = None + google_login = URL('https://accounts.google.com/(?P.+)', GoogleLoginPage) + + def __init__(self, username, password, redirect_uri, *args, **kwargs): + super(GoogleBrowser, self).__init__(username, password, *args, **kwargs) + self.redirect_uri = redirect_uri + + def do_login(self): + params = {'response_type': 'code', + 'client_id': '534890559860-r6gn7e3agcpiriehe63dkeus0tpl5i4i.apps.googleusercontent.com', + 'redirect_uri': self.redirect_uri} + + queryString = "&".join([key+'='+value for key, value in params.items()]) + self.google_login.go(auth='o/oauth2/auth', params=queryString).login(self.username, self.password) + + try: + self.code = parse_qs(urlparse(self.url).query).get('code')[0] + except: + raise BrowserIncorrectPassword() diff --git a/modules/feedly/pages.py b/modules/feedly/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..803184d06264bacf60538da3e0fef1277aad5686 --- /dev/null +++ b/modules/feedly/pages.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Bezleputh +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from datetime import datetime + +from weboob.capabilities.messages import Message +from weboob.capabilities.collection import Collection +from weboob.tools.browser2.page import JsonPage, ListElement, method, ItemElement +from weboob.tools.browser2.filters import CleanText, Dict, Format, CleanHTML + +__all__ = ['TokenPage', 'ContentsPage', 'PreferencesPage'] + + +class DictElement(ListElement): + def find_elements(self): + if self.item_xpath is not None: + for el in self.el.get(self.item_xpath): + yield el + else: + yield self.el + + +class ContentsPage(JsonPage): + + @method + class get_articles(DictElement): + item_xpath = 'items' + + class item(ItemElement): + klass = Message + + obj_id = Format(u'%s#%s', CleanText(Dict('origin/streamId')), CleanText(Dict('id'))) + obj_sender = CleanText(Dict('author', default=u'')) + obj_title = Format(u'%s - %s', CleanText(Dict('origin/title', default=u'')), CleanText(Dict('title'))) + + def obj_date(self): + return datetime.fromtimestamp(Dict('published')(self.el) / 1e3) + + def obj_content(self): + if 'content' in self.el.keys(): + return Format(u'%s%s\r\n', + CleanHTML(Dict('content/content')), CleanText(Dict('origin/htmlUrl')))(self.el) + elif 'summary' in self.el.keys(): + return Format(u'%s%s\r\n', + CleanHTML(Dict('summary/content')), CleanText(Dict('origin/htmlUrl')))(self.el) + else: + return '' + + +class TokenPage(JsonPage): + def get_token(self): + return self.doc['access_token'], self.doc['id'] + + +class EssentialsPage(JsonPage): + def get_categories(self): + for category in self.doc: + name = u'%s' % category.get('label') + yield Collection([name], name) + + def get_feeds(self, label): + for category in self.doc: + if category.get('label') == label: + feeds = category.get('subscriptions') + for feed in feeds: + yield Collection([label, feed.get('title')]) + + def get_feed_url(self, _category, _feed): + for category in self.doc: + if category.get('label') == _category: + feeds = category.get('subscriptions') + for feed in feeds: + if feed.get('title') == _feed: + return feed.get('id') + + +class PreferencesPage(JsonPage): + def get_categories(self): + for category, value in self.doc.items(): + if value in [u"shown", u"hidden"]: + yield Collection([category]) diff --git a/modules/feedly/test.py b/modules/feedly/test.py new file mode 100644 index 0000000000000000000000000000000000000000..0fd1e71bbedb5222fa33711814ef7cbb045aa389 --- /dev/null +++ b/modules/feedly/test.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Bezleputh +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from nose.plugins.skip import SkipTest +from weboob.tools.test import BackendTest + + +class FeedlyTest(BackendTest): + BACKEND = 'feedly' + + def test_feedly(self): + if self.backend.browser.username: + l1 = list(self.backend.iter_threads()) + assert len(l1) + thread = self.backend.get_thread(l1[0].id) + assert len(thread.root.content) + else: + raise SkipTest("User credentials not defined")