From 963efa032d16834540aa6c4828ca38dd667e4fc7 Mon Sep 17 00:00:00 2001 From: Roger Philibert Date: Sat, 27 Jul 2019 13:52:30 +0200 Subject: [PATCH] okc: fix authentication and website changes --- modules/okc/browser.py | 80 ++++++++++++++++++------- modules/okc/module.py | 89 +++++++++++++++++----------- modules/okc/optim/profiles_walker.py | 16 +---- 3 files changed, 118 insertions(+), 67 deletions(-) diff --git a/modules/okc/browser.py b/modules/okc/browser.py index a12efb841b..3d1d735129 100644 --- a/modules/okc/browser.py +++ b/modules/okc/browser.py @@ -17,11 +17,15 @@ # You should have received a copy of the GNU Affero General Public License # along with this weboob module. If not, see . +import re + from weboob.browser import LoginBrowser, URL -from weboob.exceptions import BrowserIncorrectPassword +from weboob.browser.browsers import DomainBrowser +from weboob.browser.pages import HTMLPage +from weboob.browser.filters.standard import CleanText +from weboob.exceptions import BrowserIncorrectPassword, ParseError from weboob.tools.json import json - __all__ = ['OkCBrowser'] @@ -33,24 +37,66 @@ def inner(browser, *args, **kwargs): return inner +class FacebookBrowser(DomainBrowser): + BASEURL = 'https://graph.facebook.com' + + access_token = None + + def login(self, username, password): + self.location('https://www.facebook.com/v2.9/dialog/oauth?app_id=484681304938818&auth_type=rerequest&channel_url=https%3A%2F%2Fstaticxx.facebook.com%2Fconnect%2Fxd_arbiter.php%3Fversion%3D44%23cb%3Df33dd8340f36618%26domain%3Dwww.okcupid.com%26origin%3Dhttps%253A%252F%252Fwww.okcupid.com%252Ff5818a5f355be8%26relation%3Dopener&client_id=484681304938818&display=popup&domain=www.okcupid.com&e2e=%7B%7D&fallback_redirect_uri=https%3A%2F%2Fwww.okcupid.com%2Flogin&locale=en_US&origin=1&redirect_uri=https%3A%2F%2Fstaticxx.facebook.com%2Fconnect%2Fxd_arbiter.php%3Fversion%3D44%23cb%3Df2ce4ca90b82cb4%26domain%3Dwww.okcupid.com%26origin%3Dhttps%253A%252F%252Fwww.okcupid.com%252Ff5818a5f355be8%26relation%3Dopener%26frame%3Df3f40f304ac5e9&response_type=token%2Csigned_request&scope=email%2Cuser_birthday%2Cuser_photos&sdk=joey&version=v2.9') + + page = HTMLPage(self, self.response) + form = page.get_form('//form[@id="login_form"]') + form['email'] = username + form['pass'] = password + self.session.headers['cookie-installing-permission'] = 'required' + self.session.cookies['wd'] = '640x1033' + self.session.cookies['act'] = '1563018648141%2F0' + form.submit(allow_redirects=False) + if 'Location' not in self.response.headers: + raise BrowserIncorrectPassword() + + self.location(self.response.headers['Location']) + + page = HTMLPage(self, self.response) + if len(page.doc.xpath('//td/div[has-class("s")]')) > 0: + raise BrowserIncorrectPassword(CleanText('//td/div[has-class("s")]')(page.doc)) + + script = page.doc.xpath('//script')[0].text + + m = re.search('access_token=([^&]+)&', script) + if m: + self.access_token = m.group(1) + else: + raise ParseError('Unable to find access_token') + + class OkCBrowser(LoginBrowser): BASEURL = 'https://www.okcupid.com' login = URL('/login') - threads = URL('/messages') - messages = URL('/apitun/messages/conversations/global_messaging') + threads = URL('/1/apitun/connections/messages/incoming') + messages = URL('/1/apitun/messages/conversations/(?P\d+)') thread_delete = URL(r'/1/apitun/messages/conversations/(?P\d+)/delete') - message_send = URL('/apitun/messages/send') + message_send = URL('/1/apitun/messages/send') quickmatch = URL(r'/quickmatch\?okc_api=1') like = URL(r'/1/apitun/profile/(?P\d+)/like') - profile = URL(r'/apitun/profile/(?P\d+)') - full_profile = URL(r'/profile/(?P.*)\?okc_api=1') + profile = URL(r'/1/apitun/profile/(?P\d+)') access_token = None me = None + def __init__(self, username, password, facebook, *args, **kwargs): + self.facebook = facebook + + super(OkCBrowser, self).__init__(username, password, *args, **kwargs) + def do_login(self): - r = self.login.go(data={'username': self.username, 'password': self.password, 'okc_api': 1}).json() + if self.facebook: + r = self.login.go(data={'facebook_access_token': self.facebook.access_token, 'okc_api': 1}).json() + + else: + r = self.login.go(data={'username': self.username, 'password': self.password, 'okc_api': 1}).json() if not 'oauth_accesstoken' in r: raise BrowserIncorrectPassword(r['status_str']) @@ -64,14 +110,13 @@ def do_login(self): self.session.headers['Authorization'] = 'Bearer %s' % self.access_token @need_login - def get_threads_list(self, folder=1): - return self.threads.go(params={'okc_api': 1, 'folder': folder, 'messages_dropdown_ajax': 1}).json() + def get_threads_list(self): + return self.threads.go().json()['data'] @need_login def get_thread_messages(self, thread_id): - r = self.messages.go(params={'access_token': self.access_token, - '_json': '{"userids":["%s"]}' % thread_id}).json() - return r[thread_id] + r = self.messages.go(thread_id=thread_id, params={'limit': 20}, headers={'endpoint_version': '2'}).json() + return r @need_login def post_message(self, thread_id, content): @@ -97,13 +142,6 @@ def find_match_profile(self): def do_rate(self, user_id): self.like.go(method='POST', user_id=user_id) - @need_login - def get_username(self, user_id): - return self.profile.go(user_id=user_id).json()['username'] - @need_login def get_profile(self, username): - if username.isdigit(): - username = self.get_username(username) - - return self.full_profile.go(username=username).json() + return self.profile.go(user_id=username).json() diff --git a/modules/okc/module.py b/modules/okc/module.py index 6825c3fc65..343287a876 100644 --- a/modules/okc/module.py +++ b/modules/okc/module.py @@ -26,9 +26,9 @@ from weboob.capabilities.messages import CapMessages, CapMessagesPost, Message, Thread from weboob.tools.backend import Module, BackendConfig from weboob.tools.misc import to_unicode -from weboob.tools.value import Value, ValueBackendPassword +from weboob.tools.value import Value, ValueBackendPassword, ValueBool -from .browser import OkCBrowser +from .browser import OkCBrowser, FacebookBrowser from .optim.profiles_walker import ProfilesWalker @@ -49,41 +49,43 @@ def set_profile(self, *args): section[key] = ProfileNode(key, key.capitalize().replace('_', ' '), value) def __init__(self, profile): - super(OkcContact, self).__init__(profile['userid'], - profile['username'], - self.STATUS_ONLINE if profile['is_online'] == '1' else self.STATUS_OFFLINE) + super(OkcContact, self).__init__(profile['user']['userid'], + profile['user']['userinfo']['displayname'], + self.STATUS_ONLINE if profile['user']['online'] else self.STATUS_OFFLINE) - self.url = 'https://www.okcupid.com/profile/%s' % self.name - self.summary = profile.get('summary', '') - self.status_msg = 'Last connection at %s' % profile['skinny']['last_online'] + self.url = 'https://www.okcupid.com/profile/%s' % self.id + self.summary = u'' + self.status_msg = profile['extras']['lastOnlineString'] - for no, photo in enumerate(profile['photos']): - self.set_photo(u'image_%i' % no, url=photo['image_url'], thumbnail_url=photo['image_url']) + for no, photo in enumerate(profile['user']['photos']): + self.set_photo(u'image_%i' % no, url=photo['full'], thumbnail_url=photo['full_small']) self.profile = OrderedDict() - self.set_profile('info', 'status', profile['status_str']) - self.set_profile('info', 'orientation', profile['orientation_str']) - self.set_profile('info', 'age', '%s yo' % profile['age']) - self.set_profile('info', 'birthday', '%04d-%02d-%02d' % (profile['birthday']['year'], profile['birthday']['month'], profile['birthday']['day'])) - self.set_profile('info', 'sex', profile['gender_str']) - self.set_profile('info', 'location', profile['location']) - self.set_profile('info', 'join_date', profile['skinny']['join_date']) - self.set_profile('stats', 'match_percent', '%s%%' % profile['matchpercentage']) - self.set_profile('stats', 'friend_percent', '%s%%' % profile['friendpercentage']) - self.set_profile('stats', 'enemy_percent', '%s%%' % profile['enemypercentage']) - for key, value in sorted(profile['skinny'].items()): - self.set_profile('details', key, value or '-') - - for essay in profile['essays']: - if len(essay['essay']) == 0: + if isinstance(profile['user']['details'], dict): + for key, label in profile['user']['details']['_labels'].items(): + self.set_profile('info', label, profile['user']['details']['values'][key]) + else: + for section in profile['user']['details']: + self.set_profile('info', section['info']['name'], section['text']['text']) + + self.set_profile('info', 'orientation', profile['user']['userinfo']['orientation']) + self.set_profile('info', 'age', '%s yo' % profile['user']['userinfo']['age']) + self.set_profile('info', 'sex', profile['user']['userinfo']['gender']) + self.set_profile('info', 'location', profile['user']['userinfo']['location']) + self.set_profile('stats', 'match_percent', '%s%%' % profile['user']['percentages']['match']) + self.set_profile('stats', 'enemy_percent', '%s%%' % profile['user']['percentages']['enemy']) + if 'friend' in profile['user']['percentages']: + self.set_profile('stats', 'friend_percent', '%s%%' % profile['user']['percentages']['friend']) + + for essay in profile['user']['essays']: + if not essay['content']: continue self.summary += '%s:\n' % essay['title'] self.summary += '-' * (len(essay['title']) + 1) self.summary += '\n' - for text in essay['essay']: - self.summary += text['rawtext'] + self.summary += essay['rawtext'] self.summary += '\n\n' self.profile['info'].flags |= ProfileNode.HEAD @@ -99,14 +101,22 @@ class OkCModule(Module, CapMessages, CapContact, CapMessagesPost, CapDating): LICENSE = 'AGPLv3+' DESCRIPTION = u'OkCupid' CONFIG = BackendConfig(Value('username', label='Username'), - ValueBackendPassword('password', label='Password')) + ValueBackendPassword('password', label='Password'), + ValueBool('facebook', label='Do you login with Facebook?', default=False)) STORAGE = {'profiles_walker': {'viewed': []}, 'sluts': {}, } BROWSER = OkCBrowser def create_default_browser(self): - return self.create_browser(self.config['username'].get(), self.config['password'].get()) + if int(self.config['facebook'].get()): + facebook = self.create_browser(klass=FacebookBrowser) + facebook.login(self.config['username'].get(), self.config['password'].get()) + else: + facebook = None + return self.create_browser(self.config['username'].get(), + self.config['password'].get(), + facebook) # ---- CapDating methods --------------------- def init_optimizations(self): @@ -120,10 +130,10 @@ def iter_threads(self): threads = self.browser.get_threads_list() for thread in threads: - t = Thread(thread['userid']) + t = Thread(thread['user']['userid']) t.flags = Thread.IS_DISCUSSION - t.title = u'Discussion with %s' % thread['user']['username'] - t.date = datetime.fromtimestamp(thread['timestamp']) + t.title = u'Discussion with %s' % thread['user']['userinfo']['displayname'] + t.date = datetime.fromtimestamp(thread['time']) yield t def get_thread(self, thread): @@ -140,7 +150,7 @@ def get_thread(self, thread): other = OkcContact(self.browser.get_profile(thread.id)) parent = None - for message in messages['messages']['messages']: + for message in messages['messages']: date = datetime.fromtimestamp(message['timestamp']) flags = 0 @@ -153,6 +163,19 @@ def get_thread(self, thread): else: receiver = other sender = me + if message.get('read', False): + flags |= Message.IS_RECEIVED + # Apply that flag on all previous messages as the 'read' + # attribute is only set on the last read message. + pmsg = parent + while pmsg: + if pmsg.flags & Message.IS_NOT_RECEIVED: + pmsg.flags |= Message.IS_RECEIVED + pmsg.flags &= ~Message.IS_NOT_RECEIVED + pmsg = pmsg.parent + else: + flags |= Message.IS_NOT_RECEIVED + msg = Message(thread=thread, id=message['id'], diff --git a/modules/okc/optim/profiles_walker.py b/modules/okc/optim/profiles_walker.py index 761bd135ed..ded695e036 100644 --- a/modules/okc/optim/profiles_walker.py +++ b/modules/okc/optim/profiles_walker.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this weboob module. If not, see . -from datetime import datetime -from dateutil.relativedelta import relativedelta from random import randint from weboob.capabilities.dating import Optimization @@ -74,13 +72,6 @@ def get_config(self): def view_profile(self): try: - # Remove old threads - for thread in self._browser.get_threads_list(folder=2): # folder 2 is the sentbox - last_message = datetime.fromtimestamp(thread['timestamp']) - if not thread['replied'] and last_message < (datetime.now() - relativedelta(months=6)): - self._logger.info('Removing old thread with %s from %s', thread['user']['username'], last_message) - self._browser.delete_thread(thread['userid']) - # Find a new profile user_id = self._browser.find_match_profile() if user_id in self._visited_profiles: @@ -89,13 +80,12 @@ def view_profile(self): self._browser.do_rate(user_id) profile = self._browser.get_profile(user_id) if self._config['first_message'] != '': - self._browser.post_message(user_id, self._config['first_message'] % {'name': profile['username']}) - self._browser.delete_thread(user_id) - self._logger.info(u'Visited profile of %s ', profile['username']) + self._browser.post_message(user_id, self._config['first_message'] % {'name': profile['user']['userinfo']['displayname']}) + self._logger.info(u'Visited profile of %s: https://www.okcupid.com/profile/%s', profile['user']['userinfo']['displayname'], profile['user']['userid']) # do not forget that we visited this profile, to avoid re-visiting it. self._visited_profiles.add(user_id) self.save() finally: if self._view_cron is not None: - self._view_cron = self._sched.schedule(randint(60, 120), self.view_profile) + self._view_cron = self._sched.schedule(randint(30, 60), self.view_profile) -- GitLab